def create_hole_msg_list(self, hole_start, hole_end, max_msg_size):
"""
Create a list of messages that can be used to fill
the "oldest" hole that the remote needs us to fill
"""
content = self.pending_out.peek(hole_start, hole_end + 1)
QE2LOG.warn('creating descriptor for hole [%d %d]', hole_start,
hole_end)
QE2LOG.warn('hole length %d content length %d',
1 + hole_end - hole_start, len(content))
# This is not very efficient in terms of memory or CPU,
# but it's not the bottleneck.
#
msgs = list()
while content:
content_prefix = content[:max_msg_size]
content = content[max_msg_size:]
msg = Qe2DataMsg(self.uuid,
content_prefix,
0,
send_offset=hole_start)
hole_start += len(content_prefix)
msgs.append(msg)
# reverse the list to make it easier to pop off the msgs.
#
msgs.reverse()
return msgs
def connectionFailed(self, reason):
"""
Could not connect to the SOCKS server
"""
QE2LOG.warn('SocksShim.connectionFailed [%s] %s', str(self),
str(reason))
def start_one(self):
"""
Take a descriptor from the latent list, and attempt
to start it
"""
if not self.latent:
QE2LOG.warn('Qe2ChannelManager.start_one: no latent channels')
return None
latent = self.latent[0]
self.latent = self.latent[1:]
QE2LOG.info('Qe2ChannelManager.start_one: starting %s', str(latent))
latent.start_time = time.time()
self.starting.append(latent)
channel = latent.create()
channel.connect()
looper = LoopingCall(channel.pusher)
channel.set_looper(looper)
looper.start(latent.loop_interval, now=True)
latent.channel = channel
return channel
def connection_failed(self, reason):
"""
Couldn't connect, or lost the connection
This will usually be subclassed, because how to react to failure
depends on details of the connection.
"""
QE2LOG.warn('Qe2Channel.connection_failed(): %s', str(reason))
def connectionFailed(self, reason=None):
"""
The app failed to connect, or something else happened.
Check: can this even happen to an endpoint?
"""
QE2LOG.warn('Qe2ClientTop.connectionFailed')
self.app_connected = False
def nudger():
man.update()
QE2LOG.info('running %d starting %d latent %d', len(man.running),
len(man.starting), len(man.latent))
QE2LOG.debug(str(man))
if len(man.running) == 0:
QE2LOG.warn('no channels running: starting latent channel')
man.start_one()
def data_recv_5(self):
"""
Handle the SOCKS responses
"""
# QE2LOG.debug('incoming len %d', len(self.shim_recv_buf))
if self.handshake_state == self.HANDSHAKE_STATE_1:
if len(self.shim_recv_buf) < 2:
return
if self.shim_recv_buf[:2] != '\x05\x00':
QE2LOG.warn('failed SOCKS5 first response')
self.transport.loseConnection()
else:
self.shim_recv_buf = self.shim_recv_buf[2:]
self.handshake_state = self.HANDSHAKE_STATE_2
self.handshake_socks5()
elif self.handshake_state == self.HANDSHAKE_STATE_2:
# NOTE: we only handle the case where the server returns
# us an ordinary IPv4 address (which is expected for a
# TCP connect request). If a server returns a DNS-type
# address, which has a variable length, this function
# can't parse it yet.
resp_len = 10
if len(self.shim_recv_buf) < resp_len:
return
response = self.shim_recv_buf[:resp_len]
self.shim_recv_buf = self.shim_recv_buf[resp_len:]
expected_prefix = '\x05\x00\x00\x01'
if response[:len(expected_prefix)] != expected_prefix:
QE2LOG.warn('failed SOCKS5 second response [prefix]')
self.transport.loseConnection()
return
# NOTE: we do not attempt to validate the returned IPv4
# address or ephemeral port. We could check them for
# basic sanity (make sure they're valid, at least) but
# we can't check that they're *correct*.
self.handshake_state = self.HANDSHAKE_COMPLETE
else:
QE2LOG.error('SocksShim.data_recv_5 unhandled state')
assert (0)
def do_connect_failed(self, reason):
"""
If we haven't already tried to reconnect too many times,
wait a brief moment and try again
"""
if self._connection_attempts < self._max_connection_attempts:
self._connection_attempts += 1
QE2LOG.debug('TCP4SocksClientEndpoint.do_connect_failed: ' +
'trying again')
self._reactor.callLater(0.5, self.do_connect)
else:
QE2LOG.warn('TCP4SocksClientEndpoint.do_connect_failed: giving up')
def handshake_socks5(self):
"""
SOCKS5 handshake
We only support a subset of SOCKS5 -- TCP, no authentication,
IPv4 addresses (if an address is given)
The SOCKS5 handshake has two states: requesting a connection
to the SOCKS server itself, and then requesting a connection
via that server. If handshake_state == 1, then this sends the
first request; otherwise it sends the second.
"""
# print 'SocksShim.handshake_socks5 %s' % str(self.handshake_state)
# TODO: check that the socks_dest_host is a string, and short
# enough to be sane.
if self.handshake_state == self.HANDSHAKE_STATE_1:
request = '\x05\x01\x00'
self.transport.write(request)
elif self.handshake_state == self.HANDSHAKE_STATE_2:
# Even though this looks like the same request that
# we send in state 1, the bytes have a different semantic
# so I don't want to combine them.
#
request = '\x05\x01\x00'
# Figure out whether the address is a dotted quad already,
# or whether we need to ask for resolution. If we try to
# convert it directly to binary, and we get a socket.error,
# then it's a DNS name.
try:
addr = socket.inet_aton(self.socks_dest_host)
except socket.error:
request += '\x03'
request += chr(len(self.socks_dest_host))
request += self.socks_dest_host
except BaseException, exc:
QE2LOG.warn('SocksShim.handshake_socks5 (2) ERR [%s]',
str(exc))
self.transport.loseConnection()
return
else:
request += '\x01'
request += addr
request += struct.pack('!H', self.socks_dest_port)
self.transport.write(request)
def is_connected(self):
"""
If the Curveball client fails, or is disconnected, then
the channel is not connected
"""
if not self.curveball_client:
return False
else:
retcode = self.curveball_client.poll()
if retcode != None:
QE2LOG.warn('curveball-client %d exited [%d]',
self.curveball_client.pid, retcode)
self.curveball_client = None
return False
return Qe2Channel.is_connected(self)
def create(self):
"""
Instantiate the channel object
"""
try:
# The endpoint might not have been created
# when the descriptor was initialized; make sure
# that it's initialized now
#
self.kwargs['endpoint'] = self.endpoint
self.channel = self.objclass(**self.kwargs)
except BaseException, exc:
QE2LOG.warn('Qe2ChannelDescriptor.create: failed [%s]', str(exc))
return None
def connectionMade(self):
"""
The app has connected.
We only permit a single connection; if we have a connection,
reject any others
"""
if self.factory.endpoint.top:
QE2LOG.warn('Qe2ClientTop.connectionMade: ' +
'rejecting second connection')
self.transport.loseConnection()
else:
QE2LOG.info('Qe2ClientTop.connectionMade: ' +
'accepting first connection')
self.factory.endpoint.top = self
self.app_connected = True
def connectionLost(self, reason=None):
"""
The app has closed its connection with us.
We need to shut down, but we need to do it in a reasonable way.
It might make sense to leave the channels open for a few moments,
and pass some chaff through, or it might make sense to shut them
down all together (i.e., if the browser exited, or the user
closed a page in the browser)
"""
QE2LOG.debug('Qe2ClientTop.connectionLost')
QE2LOG.warn('SHUTTING DOWN QUILT')
self.app_connected = False
# Try to kill any lingering channels and their subprocesses
#
if self.factory.endpoint.chanman:
self.factory.endpoint.chanman.stop_all()
def disconnect(self):
"""
Disconnect with the other end of this channel
"""
QE2LOG.info('DROPPING CURVEBALL CONNECTION')
# cleanup Curveball-specific stuff
#
if self.curveball_client:
try:
QE2LOG.info('terminating curveball-client %d',
self.curveball_client.pid)
self.curveball_client.terminate()
QE2LOG.info('waiting for curveball-client')
exit_code = self.curveball_client.wait()
QE2LOG.warn('curveball-client is dead [%d]', exit_code)
except BaseException, exc:
QE2LOG.info('curveball-client did not die [%s]', str(exc))
self.curveball_client = None
def fill_remote_hole(self):
"""
Select the "oldest" hole from remote_holes,
and create a Qe2DataMsg to fill it.
Return None if there is no hole to fill.
"""
self.prune_remote_holes()
if not self.remote_holes:
return None
(hole_start, hole_end) = sorted(self.remote_holes)[0]
# If the hole is too large to stuff into one message,
# put as much as we can
#
if (1 + hole_end - hole_start) > Qe2Msg.MAX_PAYLOAD_LEN:
hole_end = hole_start + Qe2Msg.MAX_PAYLOAD_LEN - 1
QE2LOG.debug('fill_remote_hole: filling [%d %d]', hole_start, hole_end)
try:
filler = self.pending_out.peek(hole_start, hole_end + 1)
if filler:
# print 'fill_remote_hole: filler is [%s]' % filler
hole_msg = Qe2DataMsg(self.uuid,
filler,
0,
send_offset=hole_start)
return hole_msg
else:
QE2LOG.warn('fill_remote_hole: filler [%d, %d] not found',
hole_start, hole_end)
return None
except BaseException, exc:
QE2LOG.warn('fill_remote_hole: exception [%s]', str(exc))
return None
def update(self):
"""
Update the starting, running, and latent lists to reflect
the current state of each descriptor.
The assumption is that instantiated descriptors go from
starting to running to latent as they get older. Sometimes,
however, they get stuck in starting and we have to forcibly
stop them.
"""
new_latent = self.latent[:]
new_starting = list()
new_running = list()
now = time.time()
for desc in self.starting:
if desc.channel.is_connected():
desc.start_time = None
self.running.append(desc)
elif (now - desc.start_time) > self.max_starting_wait:
QE2LOG.warn('killing channel stuck in start')
desc.channel.disconnect()
new_latent.append(desc)
else:
QE2LOG.warn('channel lingering in start state')
new_starting.append(desc)
for desc in self.running:
if desc.channel.is_connected():
new_running.append(desc)
else:
QE2LOG.info('a channel has been lost')
new_latent.append(desc)
self.latent = new_latent
self.starting = new_starting
self.running = new_running
def handle_init(self, msg):
"""
Handle receipt of a OP_INIT msg.
Note that it is an error for this to be received
by a server, so QuiltServers should subclass this method
The server_quilt_uuid is the uuid assigned to this
quilt by the server. If this is the first OP_INIT
we've gotten from the server, then remember make note
of it. If it's not the first OP_INIT, then check that
the server_quilt_uuid matches previous server_quilt_uuids,
and drop the connection if it does not.
"""
server_quilt_uuid = msg.data[:16]
if self.server_quilt_uuid == None:
QE2LOG.info('got server_quilt_uuid %s',
server_quilt_uuid.encode('hex'))
self.server_quilt_uuid = server_quilt_uuid
elif self.server_quilt_uuid != server_quilt_uuid:
QE2LOG.warn('server_quilt_uuid mismatch: expected %s got %s',
self.server_quilt_uuid.encode('hex'),
server_quilt_uuid.encode('hex'))
QE2LOG.info('dropping quilt')
# If there is a channel manager, ask it to stop
# all channels
#
if self.chanman:
QE2LOG.warn('stopping all connections')
self.chanman.stop_all()
else:
QE2LOG.error('no chanman?')
# reactor.callLater(2, reactor.stop)
reactor.stop()
def data_recv_4(self):
"""
Handle the SOCKS4 response
"""
needed = 8
if len(self.shim_recv_buf) < needed:
return
response = self.shim_recv_buf[:needed]
self.shim_recv_buf = self.shim_recv_buf[needed:]
(_null, status, _port, _ipaddr) = struct.unpack('!BBHL', response)
if status != 0x5a:
QE2LOG.warn('SOCKS4 connection failed %x %x %x', status, _ipaddr,
_port)
self.transport.loseConnection()
else:
self.handshake_state = self.HANDSHAKE_COMPLETE
return
def normalize_segments(self):
"""
Coalesce adjacent segments
"""
# print 'input ' + str(self.segments)
# Make sure the segments are in offset order
#
new_segs = sorted(self.segments, key=lambda seg: seg[FIRST_OFF_IND])
# Make sure that the segments don't overlap
# (this is just a sanity check)
for ind in xrange(len(new_segs) - 1):
if new_segs[ind][LAST_OFF_IND] > new_segs[ind + 1][FIRST_OFF_IND]:
QE2LOG.warn('WHOOPS: overlapping segments at %d [%s] and [%s]',
ind, str(new_segs[ind]), str(new_segs[ind + 1]))
# Coalesce: scan through the list of segments from start to end.
# If two segments are adjacent, set the second segment to be the
# concatenation of the two segments, and set the first to be None.
#
for ind in xrange(len(new_segs) - 1):
if (new_segs[ind][LAST_OFF_IND] == (
new_segs[ind + 1][FIRST_OFF_IND] - 1)):
new_segs[ind + 1] = [
new_segs[ind][FIRST_OFF_IND],
new_segs[ind + 1][LAST_OFF_IND],
new_segs[ind][DATA_IND] + new_segs[ind + 1][DATA_IND]
]
new_segs[ind] = None
# Remove any segments that have been absorbed.
#
new_segs = [new_seg for new_seg in new_segs if new_seg]
self.segments = new_segs
def dataReceived(self, data):
QE2LOG.debug('dataReceived marker %d', self.marker)
self.recv_buf += data
(msgs, self.recv_buf) = Qe2Msg.recv(self.recv_buf)
if not msgs:
return
# The first message on a channel MUST be an OP_CHAN message
#
# NOTE: in a full-featured channel, this message will describe the
# channel and include parameters that the server should use,
# but this is not implemented. We always use the same channel
# parameters.
if not self.uuid:
first_msg = msgs[0]
if first_msg.opcode != Qe2Msg.OP_CHAN:
QE2LOG.warn('channel started/resumed without OP_CHAN msg')
self.loseConnection()
msgs = msgs[1:]
self.uuid = first_msg.uuid
listener = self.factory.server_listener
if not (self.uuid in listener.uuid2server):
QE2LOG.info('Creating Qe2Server for uuid %s',
str(self.uuid).encode('hex'))
listener.uuid2server[self.uuid] = Qe2Server(self.uuid)
self.server = listener.uuid2server[self.uuid]
# Register ourselves with our server endpoint
#
self.server.add_bottom(self)
# We should get the parameters in the first OP_CHAN message
#
# TODO: this doesn't pay attention to the first message, but
# instead makes assumptions about the parameters
#
if not self.channel:
QE2LOG.debug('CREATING CHANNEL ON SERVER')
self.channel = Qe2SocketServerChannel(self.transport,
self.server)
# Immediately tell the client the UUID we selected for our
# local UUID, so it can tell if we crash or the connections
# are redirected.
#
# TODO: it would be better if this message didn't need to
# happen immediately (because this gives the channel a
# fingerprint) but instead was based on the channel parameters.
#
# Since we're ignoring the channel paramters in this version,
# we don't have much choice, but a smarter channel would wait.
#
resp_msg = Qe2Msg(Qe2Msg.OP_INIT, self.uuid,
self.server.local_uuid, 0)
self.transport.write(resp_msg.pack())
endpoint = self.server
endpoint.process_msgs(msgs)
def connectionFailed(self):
"""
twisted connectionFailed
"""
QE2LOG.warn('Qe2ChannelWorker: connectionFailed')
def connectionFailed(self):
QE2LOG.warn('Qe2ServerBottom.connectionFailed()')