def connectionMade(self):
"""
After the connection is established, send the SOCKS4/SOCKS4a
request to the server.
"""
# print '__SocksShim.connectionMade'
self.socks_dest_host = self.factory.socks_dest_host
self.socks_dest_port = self.factory.socks_dest_port
self.socks_dest_class = self.factory.socks_dest_class
# Decide whether to use SOCKS4 or SOCKS5 by testing
# whether we have a dotted quad or something else.
# If we already have a dotted quad address, then use
# SOCKS4; otherwise use SOCKS5 with DNS resolution
#
try:
_addr = socket.inet_aton(self.socks_dest_host)
except socket.error:
self.socks_dest_ver = 5
self.handshake_socks5()
except BaseException, exc:
QE2LOG.error('SocksSum.connectionMade() [%s]', str(exc))
self.transport.loseConnection()
def create_hole_msg_list(self, hole_start, hole_end, max_msg_size):
"""
Create a list of messages that can be used to fill
the "oldest" hole that the remote needs us to fill
"""
content = self.pending_out.peek(hole_start, hole_end + 1)
QE2LOG.warn('creating descriptor for hole [%d %d]', hole_start,
hole_end)
QE2LOG.warn('hole length %d content length %d',
1 + hole_end - hole_start, len(content))
# This is not very efficient in terms of memory or CPU,
# but it's not the bottleneck.
#
msgs = list()
while content:
content_prefix = content[:max_msg_size]
content = content[max_msg_size:]
msg = Qe2DataMsg(self.uuid,
content_prefix,
0,
send_offset=hole_start)
hole_start += len(content_prefix)
msgs.append(msg)
# reverse the list to make it easier to pop off the msgs.
#
msgs.reverse()
return msgs
def handshake_socks4(self):
"""
SOCKS4 handshake
The danted SOCKS server doesn't seem to understand SOCKS4a,
so this does not do the SOCKS4a trick of requesting DNS resolution
on the proxy.
The request has five fields:
- The protocol number (4)
- The request type (1 == TCP connection
- The destination port (2 bytes, network order)
- The destination ipaddr (4 bytes, network order)
- The nul-terminated user-ID. (1 byte, always 0)
"""
# print 'SocksShim.handshake_socks4'
request = struct.pack('!BBH', 4, 1, self.socks_dest_port)
try:
addr = socket.gethostbyname(self.socks_dest_host)
except BaseException, exc:
QE2LOG.error('SocksShim.handshake_socks4 ERROR [%s]', str(exc))
self.transport.loseConnection()
return
def connectionFailed(self, reason):
"""
Could not connect to the SOCKS server
"""
QE2LOG.warn('SocksShim.connectionFailed [%s] %s', str(self),
str(reason))
def connectionLost(self, reason=None):
"""
twisted connectionLost
"""
QE2LOG.debug('Qe2ChannelWorker: connectionLost')
self.factory.qe2chan.disconnect()
def start_one(self):
"""
Take a descriptor from the latent list, and attempt
to start it
"""
if not self.latent:
QE2LOG.warn('Qe2ChannelManager.start_one: no latent channels')
return None
latent = self.latent[0]
self.latent = self.latent[1:]
QE2LOG.info('Qe2ChannelManager.start_one: starting %s', str(latent))
latent.start_time = time.time()
self.starting.append(latent)
channel = latent.create()
channel.connect()
looper = LoopingCall(channel.pusher)
channel.set_looper(looper)
looper.start(latent.loop_interval, now=True)
latent.channel = channel
return channel
def prune_remote_holes(self):
"""
Remove any holes that were reported by the opposite endpoint
that have already filled (according to ack_recvs reported by that
endpoint). These holes might have been filled by messages
in-flight.
"""
remaining_holes = set()
for (hole_start, hole_end) in sorted(self.remote_holes):
# If the hole_end is after the remote_ack_recv, then
# this hole has an unfilled component (from what we can
# tell)
#
if hole_end > self.remote_ack_recv:
# If the start of the hole is prior to self.remote_ack_recv,
# adjust the hole_start so we don't re-send data that
# the opposite endpoint already has.
#
if hole_start <= self.remote_ack_recv:
hole_start = self.remote_ack_recv + 1
remaining_holes.add((hole_start, hole_end))
self.remote_holes = self.prune_prefix_holes(remaining_holes)
if self.remote_holes:
QE2LOG.debug('REMOTE HOLES: %s', str(self.remote_holes))
return self.remote_holes
def connectionFailed(self, reason=None):
"""
Failed to connect to the app, or some other failure
"""
QE2LOG.info('Qe2ServerTop.connectionFailed(): oops!')
self.app_connected = False
def dataReceived(self, data):
"""
Add the data to the pending queue
"""
QE2LOG.debug('Qe2ServerTop.dataReceived(): %d bytes', len(data))
self.factory.quilt_server.pending_out.enq(data)
def connectionLost(self, reason=twisted.internet.protocol.connectionDone):
"""
Connection to the SOCKS server was lost during handshake
"""
QE2LOG.error('SocksShim.connectionLost [%s] %s', str(self),
str(reason))
def connection_failed(self, reason):
"""
Couldn't connect, or lost the connection
This will usually be subclassed, because how to react to failure
depends on details of the connection.
"""
QE2LOG.warn('Qe2Channel.connection_failed(): %s', str(reason))
def __del__(self):
"""
If this reference is lost, or the process exits, then try to
terminate the curveball-client subprocess
"""
if self.curveball_client:
QE2LOG.info('Stopping curveball subprocess')
self.curveball_client.terminate()
def connectionFailed(self, reason=None):
"""
The app failed to connect, or something else happened.
Check: can this even happen to an endpoint?
"""
QE2LOG.warn('Qe2ClientTop.connectionFailed')
self.app_connected = False
def dataReceived(self, data):
"""
We have received data from the app; queue it to be sent to
the channels.
"""
QE2LOG.debug('Qe2ClientTop.dataReceived: %d bytes', len(data))
self.factory.endpoint.pending_out.enq(data)
def connectionLost(self, reason=None):
"""
server app has closed its connection with us.
TODO: we should drop all connections for this server
"""
QE2LOG.info('Qe2ServerTop.connectionLost(): YIKES!')
self.app_connected = False
def connectionMade(self):
QE2LOG.debug('Qe2ServerTop.connectionMade()')
self.factory.quilt_server.top = self
self.app_connected = True
server = None
transport = None
server = self.factory.quilt_server
transport = self.transport
def del_bottom(self, bottom):
"""
Remove a *Bottom instance from this endpoint
If the instance isn't part of this endpoint, print a diagnostic
and swallow the error
"""
try:
self.bottoms.remove(bottom)
except KeyError, exc:
QE2LOG.error('Qe2Endpoint.del_bottom: missing bottom')
def curveball_connect(self):
"""
Create a Curveball connection that we can use
to reach the quilt server.
It is an error (not reliably detected, unfortunately) to
invoke this twice for the same instance, because the ports
will collide, so if the connection has been attempted return
with failure.
"""
if self.attempted:
return None
self.attempted = True
# TODO: we need the whole, CORRECT path to the curveball client
# instead of this bogus nonsense
cb_path = CURVEBALL_CLIENT
args = list()
args.append('/usr/bin/sudo') # Only on Posix-like systems
args.append(cb_path)
args.append('-p')
args.append('%d' % self.curveball_port)
args.append('--tunnel-port')
args.append('%d' % self.curveball_internal_port)
args.append('-x')
if self.curveball_protocol == 'remora':
args.append('-r')
else:
args.append('-d')
args.append('%s:%d' % (self.decoy_host, self.decoy_port))
if self.curveball_protocol in ['http', 'http-uni']:
args.append('-w')
if self.curveball_protocol in ['http-uni', 'tls-uni']:
args.append('-u')
QE2LOG.info('Curveball command [%s]', ' '.join(args))
self.curveball_client = subprocess.Popen(args, shell=False)
QE2LOG.info('Curveball subprocess PID [%d]', self.curveball_client.pid)
# Now that we (might) be connected, set the next_ping_time
#
self.next_ping_time = time.time() + self.max_idle_time
return self.curveball_client
def do_connect_failed(self, reason):
"""
If we haven't already tried to reconnect too many times,
wait a brief moment and try again
"""
if self._connection_attempts < self._max_connection_attempts:
self._connection_attempts += 1
QE2LOG.debug('TCP4SocksClientEndpoint.do_connect_failed: ' +
'trying again')
self._reactor.callLater(0.5, self.do_connect)
else:
QE2LOG.warn('TCP4SocksClientEndpoint.do_connect_failed: giving up')
def handshake_socks5(self):
"""
SOCKS5 handshake
We only support a subset of SOCKS5 -- TCP, no authentication,
IPv4 addresses (if an address is given)
The SOCKS5 handshake has two states: requesting a connection
to the SOCKS server itself, and then requesting a connection
via that server. If handshake_state == 1, then this sends the
first request; otherwise it sends the second.
"""
# print 'SocksShim.handshake_socks5 %s' % str(self.handshake_state)
# TODO: check that the socks_dest_host is a string, and short
# enough to be sane.
if self.handshake_state == self.HANDSHAKE_STATE_1:
request = '\x05\x01\x00'
self.transport.write(request)
elif self.handshake_state == self.HANDSHAKE_STATE_2:
# Even though this looks like the same request that
# we send in state 1, the bytes have a different semantic
# so I don't want to combine them.
#
request = '\x05\x01\x00'
# Figure out whether the address is a dotted quad already,
# or whether we need to ask for resolution. If we try to
# convert it directly to binary, and we get a socket.error,
# then it's a DNS name.
try:
addr = socket.inet_aton(self.socks_dest_host)
except socket.error:
request += '\x03'
request += chr(len(self.socks_dest_host))
request += self.socks_dest_host
except BaseException, exc:
QE2LOG.warn('SocksShim.handshake_socks5 (2) ERR [%s]',
str(exc))
self.transport.loseConnection()
return
else:
request += '\x01'
request += addr
request += struct.pack('!H', self.socks_dest_port)
self.transport.write(request)
def pusher(self):
"""
A do nothing pusher: discards its data
"""
last = self.endpoint.pending_out.last
avail = last - self.endpoint.next_offset
if avail > 0:
# If there's data, we pretend we sent one
# byte by updating next_offset and ack_send,
# but we actually don't send anything.
#
QE2LOG.info('EMPTY PUSHER: throwing away one byte')
self.endpoint.next_offset += 1
self.endpoint.ack_send += 1
def connectionLost(self, reason=None):
QE2LOG.info('Qe2ServerBottom.connectionLost()')
# Deregister ourselves from our server endpoint, if we ever
# registered ourselves. (if this bottom never received any
# messages, then it won't know it's uuid and therefore can't
# register to its server)
#
if self.server:
self.server.del_bottom(self)
# If we have a looper polling for work, stop and delete it
#
if self.channel and self.channel.looper:
self.channel.looper.stop()
self.channel.looper = None
def is_connected(self):
"""
If the Curveball client fails, or is disconnected, then
the channel is not connected
"""
if not self.curveball_client:
return False
else:
retcode = self.curveball_client.poll()
if retcode != None:
QE2LOG.warn('curveball-client %d exited [%d]',
self.curveball_client.pid, retcode)
self.curveball_client = None
return False
return Qe2Channel.is_connected(self)
def connectionMade(self):
"""
The app has connected.
We only permit a single connection; if we have a connection,
reject any others
"""
if self.factory.endpoint.top:
QE2LOG.warn('Qe2ClientTop.connectionMade: ' +
'rejecting second connection')
self.transport.loseConnection()
else:
QE2LOG.info('Qe2ClientTop.connectionMade: ' +
'accepting first connection')
self.factory.endpoint.top = self
self.app_connected = True
def segment_covers(segment, hole_start, hole_last):
"""
Return a segment representing the subset of the hole that
is covered by the segment, or None of if the segment does
not overlap the hole
"""
hole_len = 1 + hole_last - hole_start
seg_start = segment[FIRST_OFF_IND]
seg_last = segment[LAST_OFF_IND]
seg_data = segment[DATA_IND]
# Need to be very careful with the extents here
#
# First check whether the hole overlaps the segment
# at all, and if not, return None.
#
# Then check whether the segment first within the hole.
#
# Finally check each of the edge cases
#
if (seg_last < hole_start) or (seg_start > hole_last):
# print 'case X'
return None
elif (seg_start <= hole_start) and (seg_last > hole_last):
start = hole_start - seg_start
last = start + hole_len
new_seg = [hole_start, hole_last, seg_data[start:last]]
# print 'case 0 hole [%d, %d] %s' % (
# hole_start, hole_last, str(new_seg))
return new_seg
elif (seg_start >= hole_start) and (seg_last <= hole_last):
return segment
elif seg_start >= hole_start:
return [seg_start, hole_last, seg_data[:1 + hole_last - seg_start]]
elif seg_last >= hole_start:
return [hole_start, seg_last, seg_data[hole_start - seg_start:]]
else:
QE2LOG.error('unhandled case')
assert (False)
def create(self):
"""
Instantiate the channel object
"""
try:
# The endpoint might not have been created
# when the descriptor was initialized; make sure
# that it's initialized now
#
self.kwargs['endpoint'] = self.endpoint
self.channel = self.objclass(**self.kwargs)
except BaseException, exc:
QE2LOG.warn('Qe2ChannelDescriptor.create: failed [%s]', str(exc))
return None
def request_hole_fill(self):
"""
If we're stuck on a hole, make a request for it
to be filled. Returns None if we're not stuck.
"""
holes = self.pending_in.find_holes()
# QE2LOG.info('my find_holes(): %s', str(holes))
if not holes:
return None
if len(holes) == 1:
if self.remote_ack_send > holes[0][0]:
hole_start = holes[0][0]
hole_end = self.remote_ack_send
else:
return None
else:
(hole_start, hole_end) = holes[0]
"""
QE2LOG.warn('RAW HOLES: %s', str(
self.prune_local_holes()
if not self.local_holes:
return None
QE2LOG.info('MY HOLES %s', str(sorted(self.local_holes)))
(hole_start, hole_end) = sorted(self.local_holes)[0]
# self.local_holes.discard((hole_start, hole_end))
"""
QE2LOG.debug('request_hole_fill: need fill for [%d %d]', hole_start,
hole_end)
msg = Qe2HoleMsg(self.uuid, (hole_start, 1 + hole_end - hole_start),
0,
0,
ack_send=self.ack_send,
ack_recv=self.ack_recv)
return msg
def __init__(self, quilt_uuid):
"""
TODO: this app_port is fictious
"""
# print 'Qe2Server.__init__()'
# Error checking: make sure that local_port is sane
super(Qe2Server, self).__init__()
app_host = Qe2Params.get('SERVER_APP_HOST')
app_port = Qe2Params.get('SERVER_APP_PORT')
# self.uuid is the uuid created for this quilt by the client
# for this quilt
#
self.uuid = quilt_uuid
# self.local_uuid is used to answer back to OP_CHAN messages;
# it identifies this endpoint uniquely, so if the quilt-server
# crashes and reboots, or some similar disruption, the client
# will know that the original endpoint has been lost
#
self.local_uuid = uuid.uuid4().bytes
self.transport = None # Needs to be initialized later
# We need to create a connection to the server app
# which will serve as our local top
self.top = None
self.top_factory = Factory()
self.top_factory.protocol = Qe2ServerTop
self.top_factory.quilt_server = self
QE2LOG.debug('Qe2ServerTop I AM CONNECTING TO (%s:%d)', str(app_host),
app_port)
endpoint = endpoints.TCP4ClientEndpoint(reactor,
app_host,
app_port,
timeout=1)
endpoint.connect(self.top_factory)
def add_remote_hole(self, new_hole_start, new_hole_end):
"""
Record that the opposite endpoint has notified us that
there is an apparent hole in their received data (by
sending us a Qe2HoleMsg).
Note that we don't just add the hole into the set verbatim;
we try to find other holes that can be combined with other
holes. (it may, in fact, be better to combine two small holes
that are separated by a non-hole in order to reduce the number
of hole-fill messages... but we don't do that right now because
the situation doesn't appear to happen in practice)
The most common case is that a hole is "extended" because
we find out that we're missing more data than we thought.
This is the most important case to handle, rather than
overlapping holes.
Holes are filled according to channel heuristics, but
recorded centrally.
"""
QE2LOG.info('add_remote_hole: adding hole (%d %d)', new_hole_start,
new_hole_end)
for (old_hole_start, old_hole_end) in sorted(self.remote_holes):
# The most common case is that a hole is "extended"
# because we find out that we're missing more data than
# we thought
#
# Another case is when a new hole fits entirely within
# an existing hole, and therefore the new hole can be ignored
#
if ((old_hole_start == new_hole_start)
and (old_hole_end < new_hole_end)):
self.remote_holes.discard((old_hole_start, old_hole_end))
self.remote_holes.add((old_hole_start, new_hole_end))
return
elif ((old_hole_start <= new_hole_start)
and (old_hole_end >= new_hole_end)):
return
self.remote_holes.add((new_hole_start, new_hole_end))
def connectionLost(self, reason=None):
"""
The app has closed its connection with us.
We need to shut down, but we need to do it in a reasonable way.
It might make sense to leave the channels open for a few moments,
and pass some chaff through, or it might make sense to shut them
down all together (i.e., if the browser exited, or the user
closed a page in the browser)
"""
QE2LOG.debug('Qe2ClientTop.connectionLost')
QE2LOG.warn('SHUTTING DOWN QUILT')
self.app_connected = False
# Try to kill any lingering channels and their subprocesses
#
if self.factory.endpoint.chanman:
self.factory.endpoint.chanman.stop_all()