def send(self, channel, cmd, data):
assert isinstance(data, binary_type)
assert len(data) <= 65535
p = struct.pack('!ccHHH', b('S'), b('S'), channel, cmd, len(data)) + data
self.outbuf.append(p)
debug2(' > channel=%d cmd=%s len=%d (fullness=%d)\n'
% (channel, cmd_to_name.get(cmd, hex(cmd)),
len(data), self.fullness))
self.fullness += len(data)
def _log(self, mux):
# Avoid double counting of the handlers. Handlers contain both udp and dns channel.
# This is only true for the server.
dns_channels = self.get_and_reset_max_dns_channels()
udp_channels = self.get_and_reset_max_udp_channels()
handlers = self.get_and_reset_max_handlers() - (dns_channels +
udp_channels)
outbufs = self.get_and_reset_max_outbufs()
fullness = self.get_and_reset_max_fullness()
too_full_flush_count = self.get_and_reset_too_full_flush_count()
mux_wrapper_buffer_size = self.get_and_reset_size_to_edge()
paused_to_the_edge_count = self.get_and_reset_paused_to_edge_count()
executed_read_count = self.get_and_reset_executed_read_count()
executed_write_count = self.get_and_reset_executed_write_count()
select_count = self.get_and_reset_select_count()
mux_not_ready_for_read_count = self.get_and_reset_mux_not_ready_for_read_count(
)
mux_not_ready_for_write_count = self.get_and_reset_mux_not_ready_for_write_count(
)
# This is needed because the dns and upd channels are on a timer. Timing might make the max dns and udp to
# be larger than handlers.
if handlers < 0:
handlers = 0
# We manipulate these values. Just guarding against mistakes.
if fullness < 0:
fullness = 0
if mux_wrapper_buffer_size < 0:
mux_wrapper_buffer_size = 0
debug1(
'handlers %d, dns_channels %d, upd_channels %d, mux_size %d, mux_outbufs %d, mux_flush_count %d, '
'size_to_edge %d, paused_to_edge_count %d, executed_read_count %d, executed_write_count %d, '
'select_count %d, mux_not_ready_for_read_count %d, mux_not_ready_for_write_count %d\n'
% (handlers, dns_channels, udp_channels, fullness, outbufs,
too_full_flush_count, mux_wrapper_buffer_size,
paused_to_the_edge_count, executed_read_count,
executed_write_count, select_count,
mux_not_ready_for_read_count, mux_not_ready_for_write_count))
p = struct.pack('!ccHHHHHHLLLLLLL', b('M'), b('M'), handlers,
dns_channels, udp_channels, outbufs,
too_full_flush_count, paused_to_the_edge_count,
fullness, mux_wrapper_buffer_size, executed_read_count,
executed_write_count, select_count,
mux_not_ready_for_read_count,
mux_not_ready_for_write_count)
mux.send(0, ssnet.CMD_METRICS, p)
def send(self, channel, cmd, data):
assert isinstance(data, binary_type)
assert len(data) <= 65535
p = struct.pack('!ccHHH', b('S'), b('S'), channel, cmd, len(data)) \
+ data
if self.outbuf and (len(data) + HDR_LEN) <= (BUFFER_SIZE - len(self.outbuf[-1])):
self.outbuf[-1] = self.outbuf[-1] + p
else:
self.outbuf.append(p)
debug2(' > channel=%d cmd=%s len=%d outbuf=%d (fullness=%d)\n'
% (channel, cmd_to_name.get(cmd, hex(cmd)),
len(data), len(self.outbuf[-1]), self.fullness))
self.fullness += len(data)
def hostwatch_ready(sock):
assert(hw.pid)
content = hw.sock.recv(4096)
if content:
lines = (hw.leftover + content).split(b('\n'))
if lines[-1]:
# no terminating newline: entry isn't complete yet!
hw.leftover = lines.pop()
lines.append(b(''))
else:
hw.leftover = b('')
mux.send(0, ssnet.CMD_HOST_LIST, b('\n').join(lines))
else:
raise Fatal('hostwatch process died')
def hostwatch_ready(sock):
assert (hw.pid)
content = hw.sock.recv(4096)
if content:
lines = (hw.leftover + content).split(b('\n'))
if lines[-1]:
# no terminating newline: entry isn't complete yet!
hw.leftover = lines.pop()
lines.append(b(''))
else:
hw.leftover = b('')
mux.send(0, ssnet.CMD_HOST_LIST, b('\n').join(lines))
else:
raise Fatal('hostwatch process died')
def __init__(self, rsock, wsock):
Handler.__init__(self, [rsock, wsock])
self.rsock = rsock
self.wsock = wsock
self.new_channel = self.got_dns_req = self.got_routes = None
self.got_udp_open = self.got_udp_data = self.got_udp_close = None
self.got_host_req = self.got_host_list = None
self.channels = {}
self.chani = 0
self.want = 0
self.inbuf = b('')
self.outbuf = []
self.fullness = 0
self.too_full = False
self.send(0, CMD_PING, b('chicken'))
def __init__(self, rsock, wsock):
Handler.__init__(self, [rsock, wsock])
self.rsock = rsock
self.wsock = wsock
self.new_channel = self.got_dns_req = self.got_routes = None
self.got_udp_open = self.got_udp_data = self.got_udp_close = None
self.got_host_req = self.got_host_list = None
self.channels = {}
self.chani = 0
self.want = 0
self.inbuf = b('')
self.outbuf = []
self.fullness = 0
self.too_full = False
self.send(0, CMD_PING, b('chicken'))
def fill(self):
if self.buf:
return
rb = self.uread()
if rb:
self.buf.append(rb)
if rb == b(''): # empty string means EOF; None means temporarily empty
self.noread()
def fill(self):
if self.buf:
return
rb = self.uread()
if rb:
self.buf.append(rb)
if rb == b(''): # empty string means EOF; None means temporarily empty
self.noread()
def handle(self):
self.fill()
# log('inbuf is: (%d,%d) %r\n'
# % (self.want, len(self.inbuf), self.inbuf))
while 1:
if len(self.inbuf) >= (self.want or HDR_LEN):
(s1, s2, channel, cmd, datalen) = \
struct.unpack('!ccHHH', self.inbuf[:HDR_LEN])
assert(s1 == b('S'))
assert(s2 == b('S'))
self.want = datalen + HDR_LEN
if self.want and len(self.inbuf) >= self.want:
data = self.inbuf[HDR_LEN:self.want]
self.inbuf = self.inbuf[self.want:]
self.want = 0
self.got_packet(channel, cmd, data)
else:
break
def handle(self):
self.fill()
# log('inbuf is: (%d,%d) %r\n'
# % (self.want, len(self.inbuf), self.inbuf))
while 1:
if len(self.inbuf) >= (self.want or HDR_LEN):
(s1, s2, channel, cmd, datalen) = \
struct.unpack('!ccHHH', self.inbuf[:HDR_LEN])
assert(s1 == b('S'))
assert(s2 == b('S'))
self.want = datalen + HDR_LEN
if self.want and len(self.inbuf) >= self.want:
data = self.inbuf[HDR_LEN:self.want]
self.inbuf = self.inbuf[self.want:]
self.want = 0
self.got_packet(channel, cmd, data)
else:
break
def callback(self, sock):
try:
data, peer = sock.recvfrom(4096)
except socket.error:
_, e = sys.exc_info()[:2]
log('UDP recv from %r port %d: %s\n' % (peer[0], peer[1], e))
return
debug2('UDP response: %d bytes\n' % len(data))
hdr = b("%s,%r," % (peer[0], peer[1]))
self.mux.send(self.chan, ssnet.CMD_UDP_DATA, hdr + data)
def callback(self, sock):
try:
data, peer = sock.recvfrom(4096)
except socket.error:
_, e = sys.exc_info()[:2]
log('UDP recv from %r port %d: %s\n' % (peer[0], peer[1], e))
return
debug2('UDP response: %d bytes\n' % len(data))
hdr = b("%s,%r," % (peer[0], peer[1]))
self.mux.send(self.chan, ssnet.CMD_UDP_DATA, hdr + data)
def fill(self):
self.rsock.setblocking(False)
try:
read = _nb_clean(os.read, self.rsock.fileno(), 32768)
except OSError:
_, e = sys.exc_info()[:2]
raise Fatal('other end: %r' % e)
# log('<<< %r\n' % b)
if read == b(''): # EOF
self.ok = False
if read:
self.inbuf += read
def uread(self):
if self.connect_to:
return None # still connecting
if self.shut_read:
return
self.rsock.setblocking(False)
try:
return _nb_clean(os.read, self.rsock.fileno(), 65536)
except OSError:
_, e = sys.exc_info()[:2]
self.seterr('uread: %s' % e)
return b('') # unexpected error... we'll call it EOF
def uread(self):
if self.connect_to:
return None # still connecting
if self.shut_read:
return
self.rsock.setblocking(False)
try:
return _nb_clean(os.read, self.rsock.fileno(), 65536)
except OSError:
_, e = sys.exc_info()[:2]
self.seterr('uread: %s' % e)
return b('') # unexpected error... we'll call it EOF
def fill(self):
self.rsock.setblocking(False)
try:
read = _nb_clean(os.read, self.rsock.fileno(), LATENCY_BUFFER_SIZE)
except OSError:
_, e = sys.exc_info()[:2]
raise Fatal('other end: %r' % e)
# log('<<< %r\n' % b)
if read == b(''): # EOF
self.ok = False
if read:
self.inbuf += read
def udp_req(channel, cmd, data):
debug2('Incoming UDP request channel=%d, cmd=%d\n' % (channel, cmd))
if cmd == ssnet.CMD_UDP_DATA:
(dstip, dstport, data) = data.split(b(','), 2)
dstport = int(dstport)
debug2('is incoming UDP data. %r %d.\n' % (dstip, dstport))
h = udphandlers[channel]
h.send((dstip, dstport), data)
elif cmd == ssnet.CMD_UDP_CLOSE:
debug2('is incoming UDP close\n')
h = udphandlers[channel]
h.ok = False
del mux.channels[channel]
def udp_req(channel, cmd, data):
debug2('Incoming UDP request channel=%d, cmd=%d\n' % (channel, cmd))
if cmd == ssnet.CMD_UDP_DATA:
(dstip, dstport, data) = data.split(b(','), 2)
dstport = int(dstport)
debug2('is incoming UDP data. %r %d.\n' % (dstip, dstport))
h = udphandlers[channel]
h.send((dstip, dstport), data)
elif cmd == ssnet.CMD_UDP_CLOSE:
debug2('is incoming UDP close\n')
h = udphandlers[channel]
h.ok = False
del mux.channels[channel]
def maybe_resume(self, wrote):
self.buf_total = self.buf_total - wrote
global _global_mux_wrapper_buffer_size
_global_mux_wrapper_buffer_size -= wrote
debug3(
'Global mux wrap buf size: %d. Individual buf size: %d. On channel %s\n'
% (_global_mux_wrapper_buffer_size, self.buf_total, self.channel))
if self.isPaused and self.buf_total < INDIVIDUAL_BUFFER_RESUME_SIZE:
self.mux.send(self.channel, CMD_RESUME, b(''))
self.isPaused = False
log('Global mux wrap buf size: %d. Individual buf size: %d (below threshold). Sent CMD_RESUME on channel %s\n'
% (_global_mux_wrapper_buffer_size, self.buf_total,
self.channel))
def fill(self):
try:
os.set_blocking(self.rfile.fileno(), False)
except AttributeError:
# python < 3.5
flags = fcntl.fcntl(self.rfile.fileno(), fcntl.F_GETFL)
flags |= os.O_NONBLOCK
flags = fcntl.fcntl(self.rfile.fileno(), fcntl.F_SETFL, flags)
try:
read = _nb_clean(os.read, self.rfile.fileno(), LATENCY_BUFFER_SIZE)
except OSError:
_, e = sys.exc_info()[:2]
raise Fatal('other end: %r' % e)
# log('<<< %r' % b)
if read == b(''): # EOF
self.ok = False
if read:
self.inbuf += read
def maybe_pause(self, data):
self.buf_total = self.buf_total + len(data)
global _global_mux_wrapper_buffer_size
_global_mux_wrapper_buffer_size += len(data)
self.metrics.save_max_mux_wrapper_buffer_size(
_global_mux_wrapper_buffer_size)
debug3(
'Global mux wrap buf size: %d. Individual buf size: %d. On channel %s\n'
% (_global_mux_wrapper_buffer_size, self.buf_total, self.channel))
if not self.isPaused and _global_mux_wrapper_buffer_size > PAUSE_TRAFFIC_TO_EDGE_THRESHOLD and self.buf_total\
> INDIVIDUAL_BUFFER_PAUSE_SIZE:
self.mux.send(self.channel, CMD_PAUSE, b(''))
self.isPaused = True
self.metrics.incr_paused_to_the_edge_count()
log('Global mux wrap buf size: %d (above threshold). Individual buf size: %d (above threshold). Sent CMD_PAUSE on channel %s\n'
% (_global_mux_wrapper_buffer_size, self.buf_total,
self.channel))
def fill(self):
try:
os.set_blocking(self.rfile.fileno(), False)
except AttributeError:
# python < 3.5
flags = fcntl.fcntl(self.rfile.fileno(), fcntl.F_GETFL)
flags |= os.O_NONBLOCK
flags = fcntl.fcntl(self.rfile.fileno(), fcntl.F_SETFL, flags)
try:
# If LATENCY_BUFFER_SIZE is inappropriately large, we will
# get a MemoryError here. Read no more than 1MiB.
read = _nb_clean(os.read, self.rfile.fileno(),
min(1048576, LATENCY_BUFFER_SIZE))
except OSError:
_, e = sys.exc_info()[:2]
raise Fatal('other end: %r' % e)
# log('<<< %r' % b)
if read == b(''): # EOF
self.ok = False
if read:
self.inbuf += read
def uread(self):
if self.shut_read:
return b('') # EOF
else:
return None # no data available right now
def nowrite(self):
if not self.shut_write:
self.mux.send(self.channel, CMD_TCP_EOF, b(''))
self.setnowrite()
def noread(self):
if not self.shut_read:
self.mux.send(self.channel, CMD_TCP_STOP_SENDING, b(''))
self.setnoread()
def noread(self):
if not self.shut_read:
debug2('%r: done reading\n' % self)
self.shut_read = True
self.mux.send(self.channel, CMD_TCP_STOP_SENDING, b(''))
self.maybe_close()
def noread(self):
if not self.shut_read:
debug2('%r: done reading\n' % self)
self.shut_read = True
self.mux.send(self.channel, CMD_TCP_STOP_SENDING, b(''))
self.maybe_close()
def nowrite(self):
if not self.shut_write:
self.mux.send(self.channel, CMD_TCP_EOF, b(''))
self.setnowrite()
def main(latency_control):
debug1('Starting server with Python version %s\n'
% platform.python_version())
if helpers.verbose >= 1:
helpers.logprefix = ' s: '
else:
helpers.logprefix = 'server: '
debug1('latency control setting = %r\n' % latency_control)
routes = list(list_routes())
debug1('available routes:\n')
for r in routes:
debug1(' %d/%s/%d\n' % r)
# synchronization header
sys.stdout.write('\0\0SSHUTTLE0001')
sys.stdout.flush()
handlers = []
mux = Mux(socket.fromfd(sys.stdin.fileno(),
socket.AF_INET, socket.SOCK_STREAM),
socket.fromfd(sys.stdout.fileno(),
socket.AF_INET, socket.SOCK_STREAM))
handlers.append(mux)
routepkt = ''
for r in routes:
routepkt += '%d,%s,%d\n' % r
mux.send(0, ssnet.CMD_ROUTES, b(routepkt))
hw = Hostwatch()
hw.leftover = b('')
def hostwatch_ready(sock):
assert(hw.pid)
content = hw.sock.recv(4096)
if content:
lines = (hw.leftover + content).split(b('\n'))
if lines[-1]:
# no terminating newline: entry isn't complete yet!
hw.leftover = lines.pop()
lines.append(b(''))
else:
hw.leftover = b('')
mux.send(0, ssnet.CMD_HOST_LIST, b('\n').join(lines))
else:
raise Fatal('hostwatch process died')
def got_host_req(data):
if not hw.pid:
(hw.pid, hw.sock) = start_hostwatch(data.strip().split())
handlers.append(Handler(socks=[hw.sock],
callback=hostwatch_ready))
mux.got_host_req = got_host_req
def new_channel(channel, data):
(family, dstip, dstport) = data.decode("ASCII").split(',', 2)
family = int(family)
dstport = int(dstport)
outwrap = ssnet.connect_dst(family, dstip, dstport)
handlers.append(Proxy(MuxWrapper(mux, channel), outwrap))
mux.new_channel = new_channel
dnshandlers = {}
def dns_req(channel, data):
debug2('Incoming DNS request channel=%d.\n' % channel)
h = DnsProxy(mux, channel, data)
handlers.append(h)
dnshandlers[channel] = h
mux.got_dns_req = dns_req
udphandlers = {}
def udp_req(channel, cmd, data):
debug2('Incoming UDP request channel=%d, cmd=%d\n' % (channel, cmd))
if cmd == ssnet.CMD_UDP_DATA:
(dstip, dstport, data) = data.split(",", 2)
dstport = int(dstport)
debug2('is incoming UDP data. %r %d.\n' % (dstip, dstport))
h = udphandlers[channel]
h.send((dstip, dstport), data)
elif cmd == ssnet.CMD_UDP_CLOSE:
debug2('is incoming UDP close\n')
h = udphandlers[channel]
h.ok = False
del mux.channels[channel]
def udp_open(channel, data):
debug2('Incoming UDP open.\n')
family = int(data)
mux.channels[channel] = lambda cmd, data: udp_req(channel, cmd, data)
if channel in udphandlers:
raise Fatal('UDP connection channel %d already open' % channel)
else:
h = UdpProxy(mux, channel, family)
handlers.append(h)
udphandlers[channel] = h
mux.got_udp_open = udp_open
while mux.ok:
if hw.pid:
assert(hw.pid > 0)
(rpid, rv) = os.waitpid(hw.pid, os.WNOHANG)
if rpid:
raise Fatal(
'hostwatch exited unexpectedly: code 0x%04x\n' % rv)
ssnet.runonce(handlers, mux)
if latency_control:
mux.check_fullness()
if dnshandlers:
now = time.time()
remove = []
for channel, h in dnshandlers.items():
if h.timeout < now or not h.ok:
debug3('expiring dnsreqs channel=%d\n' % channel)
remove.append(channel)
h.ok = False
for channel in remove:
del dnshandlers[channel]
if udphandlers:
remove = []
for channel, h in udphandlers.items():
if not h.ok:
debug3('expiring UDP channel=%d\n' % channel)
remove.append(channel)
h.ok = False
for channel in remove:
del udphandlers[channel]
def check_fullness(self):
if self.fullness > 32768:
if not self.too_full:
self.send(0, CMD_PING, b('rttest'))
self.too_full = True
def check_fullness(self):
if self.fullness > 32768:
if not self.too_full:
self.send(0, CMD_PING, b('rttest'))
self.too_full = True
def nowrite(self):
if not self.shut_write:
debug2('%r: done writing\n' % self)
self.shut_write = True
self.mux.send(self.channel, CMD_TCP_EOF, b(''))
self.maybe_close()
def noread(self):
if not self.shut_read:
self.mux.send(self.channel, CMD_TCP_STOP_SENDING, b(''))
self.setnoread()
def main(latency_control, auto_hosts):
debug1('Starting server with Python version %s\n' %
platform.python_version())
if helpers.verbose >= 1:
helpers.logprefix = ' s: '
else:
helpers.logprefix = 'server: '
debug1('latency control setting = %r\n' % latency_control)
routes = list(list_routes())
debug1('available routes:\n')
for r in routes:
debug1(' %d/%s/%d\n' % r)
# synchronization header
sys.stdout.write('\0\0SSHUTTLE0001')
sys.stdout.flush()
handlers = []
mux = Mux(
socket.fromfd(sys.stdin.fileno(), socket.AF_INET, socket.SOCK_STREAM),
socket.fromfd(sys.stdout.fileno(), socket.AF_INET, socket.SOCK_STREAM))
handlers.append(mux)
routepkt = ''
for r in routes:
routepkt += '%d,%s,%d\n' % r
mux.send(0, ssnet.CMD_ROUTES, b(routepkt))
hw = Hostwatch()
hw.leftover = b('')
def hostwatch_ready(sock):
assert (hw.pid)
content = hw.sock.recv(4096)
if content:
lines = (hw.leftover + content).split(b('\n'))
if lines[-1]:
# no terminating newline: entry isn't complete yet!
hw.leftover = lines.pop()
lines.append(b(''))
else:
hw.leftover = b('')
mux.send(0, ssnet.CMD_HOST_LIST, b('\n').join(lines))
else:
raise Fatal('hostwatch process died')
def got_host_req(data):
if not hw.pid:
(hw.pid, hw.sock) = start_hostwatch(data.strip().split(),
auto_hosts)
handlers.append(Handler(socks=[hw.sock], callback=hostwatch_ready))
mux.got_host_req = got_host_req
def new_channel(channel, data):
(family, dstip, dstport) = data.decode("ASCII").split(',', 2)
family = int(family)
# AF_INET is the same constant on Linux and BSD but AF_INET6
# is different. As the client and server can be running on
# different platforms we can not just set the socket family
# to what comes in the wire.
if family != socket.AF_INET:
family = socket.AF_INET6
dstport = int(dstport)
outwrap = ssnet.connect_dst(family, dstip, dstport)
handlers.append(Proxy(MuxWrapper(mux, channel), outwrap))
mux.new_channel = new_channel
dnshandlers = {}
def dns_req(channel, data):
debug2('Incoming DNS request channel=%d.\n' % channel)
h = DnsProxy(mux, channel, data)
handlers.append(h)
dnshandlers[channel] = h
mux.got_dns_req = dns_req
udphandlers = {}
def udp_req(channel, cmd, data):
debug2('Incoming UDP request channel=%d, cmd=%d\n' % (channel, cmd))
if cmd == ssnet.CMD_UDP_DATA:
(dstip, dstport, data) = data.split(b(','), 2)
dstport = int(dstport)
debug2('is incoming UDP data. %r %d.\n' % (dstip, dstport))
h = udphandlers[channel]
h.send((dstip, dstport), data)
elif cmd == ssnet.CMD_UDP_CLOSE:
debug2('is incoming UDP close\n')
h = udphandlers[channel]
h.ok = False
del mux.channels[channel]
def udp_open(channel, data):
debug2('Incoming UDP open.\n')
family = int(data)
mux.channels[channel] = lambda cmd, data: udp_req(channel, cmd, data)
if channel in udphandlers:
raise Fatal('UDP connection channel %d already open' % channel)
else:
h = UdpProxy(mux, channel, family)
handlers.append(h)
udphandlers[channel] = h
mux.got_udp_open = udp_open
while mux.ok:
if hw.pid:
assert (hw.pid > 0)
(rpid, rv) = os.waitpid(hw.pid, os.WNOHANG)
if rpid:
raise Fatal('hostwatch exited unexpectedly: code 0x%04x\n' %
rv)
ssnet.runonce(handlers, mux)
if latency_control:
mux.check_fullness()
if dnshandlers:
now = time.time()
remove = []
for channel, h in dnshandlers.items():
if h.timeout < now or not h.ok:
debug3('expiring dnsreqs channel=%d\n' % channel)
remove.append(channel)
h.ok = False
for channel in remove:
del dnshandlers[channel]
if udphandlers:
remove = []
for channel, h in udphandlers.items():
if not h.ok:
debug3('expiring UDP channel=%d\n' % channel)
remove.append(channel)
h.ok = False
for channel in remove:
del udphandlers[channel]
def check_fullness(self):
if self.fullness > LATENCY_BUFFER_SIZE:
if not self.too_full:
self.send(0, CMD_PING, b('rttest'))
self.too_full = True
def uread(self):
if self.shut_read:
return b('') # EOF
else:
return None # no data available right now
def nowrite(self):
if not self.shut_write:
debug2('%r: done writing\n' % self)
self.shut_write = True
self.mux.send(self.channel, CMD_TCP_EOF, b(''))
self.maybe_close()