def soft_shutdown(signal_number, stack_frame):
"""
Attempt to cleanly stop the running API processes.
The socket and HTTP servers are continuously running processes, so we cannot simply join() on them and wait for
them to finish. Instead, clean up as we can and then terminate the procs.
Note that a sigkill cannot be caught.
Flask/HTTP close process:
- stop pulling work off of the queue
- let currently running methods finish.
- terminate proc
Socket shutdown process:
- shutdown() to stop rx/tx, sending FIN/EOF.
- close() to decrement a handle counter to the socket. if socket was shutdown, this should mean deallocation
Shutdown options:
0 - no more rx (SHUT_RD)
1 - no more tx (SHUT_WR)
2 - no more rx/tx (SHUT_RDWR)
More verbosely:
1) server "no more data will be sent" .shutdown(1)
2) client receives and finishes sending its data to server, who is still listening for N duration
3) client "no more data will be sent" .shutdown(1) -- could also optionally SHUT_RD or close at this point
4) server receives #3 from all clients, can do a shutdown(2) if wanted or just close socket
:param signal_number: signal integer from os signal
:param stack_frame: stack frame at the top at the time of the signal
:return:
"""
log.info("Caught shutdown (code %s). Attempting to gracefully close." % signal_number)
def close_socket():
log.debug("Setting poison pill to shut down the socket.")
tcpsocket.POISON_PILL = True
def soft_shutdown(signal_number, stack_frame):
"""
Attempt to cleanly stop the running API processes.
The socket and HTTP servers are continuously running processes, so we cannot simply join() on them and wait for
them to finish. Instead, clean up as we can and then terminate the procs.
Note that a sigkill cannot be caught.
Flask/HTTP close process:
- stop pulling work off of the queue
- let currently running methods finish.
- terminate proc
Socket shutdown process:
- shutdown() to stop rx/tx, sending FIN/EOF.
- close() to decrement a handle counter to the socket. if socket was shutdown, this should mean deallocation
Shutdown options:
0 - no more rx (SHUT_RD)
1 - no more tx (SHUT_WR)
2 - no more rx/tx (SHUT_RDWR)
More verbosely:
1) server "no more data will be sent" .shutdown(1)
2) client receives and finishes sending its data to server, who is still listening for N duration
3) client "no more data will be sent" .shutdown(1) -- could also optionally SHUT_RD or close at this point
4) server receives #3 from all clients, can do a shutdown(2) if wanted or just close socket
:param signal_number: signal integer from os signal
:param stack_frame: stack frame at the top at the time of the signal
:return:
"""
log.info("Caught shutdown (code %s). Attempting to gracefully close." % signal_number)
def close_socket():
log.debug("Setting poison pill to shut down the socket.")
tcpsocket.POISON_PILL = True
def serve(POISON_PILL):
sock_tcp = None
rx_sockets = []
tx_sockets = []
err_sockets = []
while True:
# build a connection if we don't have one. check none specifically. don't make this a complex with
# POISON_PILL -- it's possible we may need to have a listener for clients to drain their buffers
if sock_tcp is None:
sock_tcp = make_server_socket()
try:
# rlist - sockets to try reading; return from select is subset of actually readable (has data in buffer)
# wlist - sockets to try writing to; return from select is subset of actually writable (outbound buffer available)
# xlist - sockets to check for errors; return from select is subset of actually errored
# select is blocking, so give it a timeout
# "server" goes to rlist
# socket to .connect() to clients gets appended into the wlist
# see man select for more details
try:
rx_sockets, tx_sockets, err_sockets = select.select(SOCKET_CONNECTIONS, [], [], API.SOCKET_TIMEOUT_SECONDS)
except KeyboardInterrupt:
POISON_PILL = True
except select.error as e:
log.fatal("Could not select() on sockets. Closing socket. %s " % e)
# break here will jump to the close
break
# Please double-check the logic here
# check for poison pill. bool is very slightly faster than constant int 1 comparison in py3.5
if POISON_PILL:
log.info("Received poison pill for shutdown.")
# if we didn't have any clients between startup and shutdown, rx_sockets will be empty.
if rx_sockets:
for sock in rx_sockets:
try:
sock.shutdown(1)
# give it a little time for the socket to process the shutdown
time.sleep(.2)
sock.close()
except OSError:
# socket probably wasn't connected at .shutdown()
pass
SOCKET_CONNECTIONS.remove(sock)
raise SystemExit('YAMS Socket Server exiting due to a poison pill.')
# to respond to conversation initiated by a client (recv())
for sock in rx_sockets:
if sock == sock_tcp:
# The server socket is responsible for accepting incoming connections and responding with its own client
# connection socket. We know we have a new client conn when data comes into our server socket buffer.
# There shouldn't be an issue with this strategy for select/async with non-blocking sockets, but if so, this
# would be where to dispatch a new thread/native thread
socket_fd, address = sock_tcp.accept()
SOCKET_CONNECTIONS.append(socket_fd)
log.info("Client (%s, %s) connected" % address)
else:
# rx'd from a client
try:
# message from the client here
rx_data = sock.recv(API.SOCKET_RX_BUFFER_BYTES)
# no exception, we have an rx_data
if rx_data:
# todo: process the data. if way too large, disconnect client -- something is wrong..
# socket.send returns the number of bytes sent, so this should go to the metrics gather-er
# sendall returns None on success. there is no way to determine how much data was sent on err, count bytes for tx size
# check for quit messages, evaluate based on the different yams event types
# socket.send returns the number of bytes sent, so this should go to the metrics gather-er
# sendall returns None on success. there is no way to determine how much data was sent on err, count bytes for tx size
if not API.DEBUG:
# 1) Include zlib library and metadata about the request (+= zlib.ZLIB_VERSION)
# e.g. metadata: zlib=1.2.5
# 2) Checksum and store in checksum field (md5 or fletchers? fast, cheap - SSL does the security)
# 3) Compress (zlib.compress())
# 4) Encrypt
# with this order, if there's a corruption over the wire, zlib will see it as corrupted data
# then we'll have the checksum to compare the uncompressed payload against.
pass
# Else: Checksum, store checksum in message
sock.send(bytes_encoder("RECEIVED DATA: %s" % rx_data))
# Probably errored on rx_data -- did something else deplete our buffer?
# this next exception type is an educated guess -- couldn't get timing down for broken socket (e.g. ^c)
except select.error as e:
# Probably fine that the connection is no longer there after initial connection.
# see man 2 accept
log.debug(str(e))
log.warn("Client socket <%s> is not connected." % sock)
# this could be harmless, but could be a sign of unintended clamping of client conns
# just close the conn and then remove it from our candidate FDs. don't bother with .shutdown(), but
# do close explicitly instead of assuming it will get GC'd
sock.close()
try:
log.info("Removing a socket due to a select error.")
SOCKET_CONNECTIONS.remove(sock)
except ValueError:
# sock wasn't in socket_connections. that's what we want, so that's fine.
continue
# to initiate a conversation with a client (connect())
for socket in tx_sockets:
pass
for socket in err_sockets:
pass
if API.DEBUG:
log.debug("rlist: %s" % rx_sockets)
log.debug("wlist: %s" % tx_sockets)
log.debug("xlist: %s" % err_sockets)
time.sleep(API.SOCKET_THREAD_YIELD_EPSILON)
# poll for server shutdown on some interval too. if we reach shutdown, tear down connections from clients
# no longer True, exception raised, or we exited
sock_tcp.close()
"""
log.info("Caught shutdown (code %s). Attempting to gracefully close." % signal_number)
def close_socket():
log.debug("Setting poison pill to shut down the socket.")
tcpsocket.POISON_PILL = True
# for socket in rx_socket, wx_socket, err_sockets, shutdown
# wait N duration
# join
if __name__ == "__main__":
log.info("Starting up.")
if API.DEBUG:
flask_reloader_sock_msg = "You will be receiving the error 'Failed to create socket: [Errno 48]...' " \
"this is typical and due to Flask's reloader that gets called in debug mode."
log.critical(flask_reloader_sock_msg)
print(flask_reloader_sock_msg)
# spawn a fresh process -- we don't have resources that we need to share via fork().
# use a context in case our user wants to further subprocess. check ipcs after failure mode testing
# to make sure that this isn't problematic for named semaphores.
_context = _mp.get_context('spawn')
# create a general purpose internal queue -- useful for important messages like 'disconnect client x'.
# unfortunately, POSIX semaphores require r/w to /dev/shm, which isn't there on OS X.
# Leave here as a reminder that this functionality should be implemented.
"""
log.info("Caught shutdown (code %s). Attempting to gracefully close." % signal_number)
def close_socket():
log.debug("Setting poison pill to shut down the socket.")
tcpsocket.POISON_PILL = True
# for socket in rx_socket, wx_socket, err_sockets, shutdown
# wait N duration
# join
if __name__ == "__main__":
log.info("Starting up.")
if API.DEBUG:
flask_reloader_sock_msg = (
"You will be receiving the error 'Failed to create socket: [Errno 48]...' "
"this is typical and due to Flask's reloader that gets called in debug mode."
)
log.critical(flask_reloader_sock_msg)
print(flask_reloader_sock_msg)
# spawn a fresh process -- we don't have resources that we need to share via fork().
# use a context in case our user wants to further subprocess. check ipcs after failure mode testing
# to make sure that this isn't problematic for named semaphores.
_context = _mp.get_context("spawn")
# create a general purpose internal queue -- useful for important messages like 'disconnect client x'.
def serve(POISON_PILL):
sock_tcp = None
rx_sockets = []
tx_sockets = []
err_sockets = []
while True:
# build a connection if we don't have one. check none specifically. don't make this a complex with
# POISON_PILL -- it's possible we may need to have a listener for clients to drain their buffers
if sock_tcp is None:
sock_tcp = make_server_socket()
try:
# rlist - sockets to try reading; return from select is subset of actually readable (has data in buffer)
# wlist - sockets to try writing to; return from select is subset of actually writable (outbound buffer available)
# xlist - sockets to check for errors; return from select is subset of actually errored
# select is blocking, so give it a timeout
# "server" goes to rlist
# socket to .connect() to clients gets appended into the wlist
# see man select for more details
try:
rx_sockets, tx_sockets, err_sockets = select.select(
SOCKET_CONNECTIONS, [], [], API.SOCKET_TIMEOUT_SECONDS)
except KeyboardInterrupt:
POISON_PILL = True
except select.error as e:
log.fatal("Could not select() on sockets. Closing socket. %s " %
e)
# break here will jump to the close
break
# Please double-check the logic here
# check for poison pill. bool is very slightly faster than constant int 1 comparison in py3.5
if POISON_PILL:
log.info("Received poison pill for shutdown.")
# if we didn't have any clients between startup and shutdown, rx_sockets will be empty.
if rx_sockets:
for sock in rx_sockets:
try:
sock.shutdown(1)
# give it a little time for the socket to process the shutdown
time.sleep(.2)
sock.close()
except OSError:
# socket probably wasn't connected at .shutdown()
pass
SOCKET_CONNECTIONS.remove(sock)
raise SystemExit(
'YAMS Socket Server exiting due to a poison pill.')
# to respond to conversation initiated by a client (recv())
for sock in rx_sockets:
if sock == sock_tcp:
# The server socket is responsible for accepting incoming connections and responding with its own client
# connection socket. We know we have a new client conn when data comes into our server socket buffer.
# There shouldn't be an issue with this strategy for select/async with non-blocking sockets, but if so, this
# would be where to dispatch a new thread/native thread
socket_fd, address = sock_tcp.accept()
SOCKET_CONNECTIONS.append(socket_fd)
log.info("Client (%s, %s) connected" % address)
else:
# rx'd from a client
try:
# message from the client here
rx_data = sock.recv(API.SOCKET_RX_BUFFER_BYTES)
# no exception, we have an rx_data
if rx_data:
# todo: process the data. if way too large, disconnect client -- something is wrong..
# socket.send returns the number of bytes sent, so this should go to the metrics gather-er
# sendall returns None on success. there is no way to determine how much data was sent on err, count bytes for tx size
# check for quit messages, evaluate based on the different yams event types
# socket.send returns the number of bytes sent, so this should go to the metrics gather-er
# sendall returns None on success. there is no way to determine how much data was sent on err, count bytes for tx size
if not API.DEBUG:
# 1) Include zlib library and metadata about the request (+= zlib.ZLIB_VERSION)
# e.g. metadata: zlib=1.2.5
# 2) Checksum and store in checksum field (md5 or fletchers? fast, cheap - SSL does the security)
# 3) Compress (zlib.compress())
# 4) Encrypt
# with this order, if there's a corruption over the wire, zlib will see it as corrupted data
# then we'll have the checksum to compare the uncompressed payload against.
pass
# Else: Checksum, store checksum in message
sock.send(bytes_encoder("RECEIVED DATA: %s" % rx_data))
# Probably errored on rx_data -- did something else deplete our buffer?
# this next exception type is an educated guess -- couldn't get timing down for broken socket (e.g. ^c)
except select.error as e:
# Probably fine that the connection is no longer there after initial connection.
# see man 2 accept
log.debug(str(e))
log.warn("Client socket <%s> is not connected." % sock)
# this could be harmless, but could be a sign of unintended clamping of client conns
# just close the conn and then remove it from our candidate FDs. don't bother with .shutdown(), but
# do close explicitly instead of assuming it will get GC'd
sock.close()
try:
log.info("Removing a socket due to a select error.")
SOCKET_CONNECTIONS.remove(sock)
except ValueError:
# sock wasn't in socket_connections. that's what we want, so that's fine.
continue
# to initiate a conversation with a client (connect())
for socket in tx_sockets:
pass
for socket in err_sockets:
pass
if API.DEBUG:
log.debug("rlist: %s" % rx_sockets)
log.debug("wlist: %s" % tx_sockets)
log.debug("xlist: %s" % err_sockets)
time.sleep(API.SOCKET_THREAD_YIELD_EPSILON)
# poll for server shutdown on some interval too. if we reach shutdown, tear down connections from clients
# no longer True, exception raised, or we exited
sock_tcp.close()
