def start_one(self):
"""
Take a descriptor from the latent list, and attempt
to start it
"""
if not self.latent:
QE2LOG.warn('Qe2ChannelManager.start_one: no latent channels')
return None
latent = self.latent[0]
self.latent = self.latent[1:]
QE2LOG.info('Qe2ChannelManager.start_one: starting %s', str(latent))
latent.start_time = time.time()
self.starting.append(latent)
channel = latent.create()
channel.connect()
looper = LoopingCall(channel.pusher)
channel.set_looper(looper)
looper.start(latent.loop_interval, now=True)
latent.channel = channel
return channel
def connectionFailed(self, reason=None):
"""
Failed to connect to the app, or some other failure
"""
QE2LOG.info('Qe2ServerTop.connectionFailed(): oops!')
self.app_connected = False
def __del__(self):
"""
If this reference is lost, or the process exits, then try to
terminate the curveball-client subprocess
"""
if self.curveball_client:
QE2LOG.info('Stopping curveball subprocess')
self.curveball_client.terminate()
def connectionLost(self, reason=None):
"""
server app has closed its connection with us.
TODO: we should drop all connections for this server
"""
QE2LOG.info('Qe2ServerTop.connectionLost(): YIKES!')
self.app_connected = False
def nudger():
man.update()
QE2LOG.info('running %d starting %d latent %d', len(man.running),
len(man.starting), len(man.latent))
QE2LOG.debug(str(man))
if len(man.running) == 0:
QE2LOG.warn('no channels running: starting latent channel')
man.start_one()
def curveball_connect(self):
"""
Create a Curveball connection that we can use
to reach the quilt server.
It is an error (not reliably detected, unfortunately) to
invoke this twice for the same instance, because the ports
will collide, so if the connection has been attempted return
with failure.
"""
if self.attempted:
return None
self.attempted = True
# TODO: we need the whole, CORRECT path to the curveball client
# instead of this bogus nonsense
cb_path = CURVEBALL_CLIENT
args = list()
args.append('/usr/bin/sudo') # Only on Posix-like systems
args.append(cb_path)
args.append('-p')
args.append('%d' % self.curveball_port)
args.append('--tunnel-port')
args.append('%d' % self.curveball_internal_port)
args.append('-x')
if self.curveball_protocol == 'remora':
args.append('-r')
else:
args.append('-d')
args.append('%s:%d' % (self.decoy_host, self.decoy_port))
if self.curveball_protocol in ['http', 'http-uni']:
args.append('-w')
if self.curveball_protocol in ['http-uni', 'tls-uni']:
args.append('-u')
QE2LOG.info('Curveball command [%s]', ' '.join(args))
self.curveball_client = subprocess.Popen(args, shell=False)
QE2LOG.info('Curveball subprocess PID [%d]', self.curveball_client.pid)
# Now that we (might) be connected, set the next_ping_time
#
self.next_ping_time = time.time() + self.max_idle_time
return self.curveball_client
def pusher(self):
"""
A do nothing pusher: discards its data
"""
last = self.endpoint.pending_out.last
avail = last - self.endpoint.next_offset
if avail > 0:
# If there's data, we pretend we sent one
# byte by updating next_offset and ack_send,
# but we actually don't send anything.
#
QE2LOG.info('EMPTY PUSHER: throwing away one byte')
self.endpoint.next_offset += 1
self.endpoint.ack_send += 1
def connectionLost(self, reason=None):
QE2LOG.info('Qe2ServerBottom.connectionLost()')
# Deregister ourselves from our server endpoint, if we ever
# registered ourselves. (if this bottom never received any
# messages, then it won't know it's uuid and therefore can't
# register to its server)
#
if self.server:
self.server.del_bottom(self)
# If we have a looper polling for work, stop and delete it
#
if self.channel and self.channel.looper:
self.channel.looper.stop()
self.channel.looper = None
def connectionMade(self):
"""
The app has connected.
We only permit a single connection; if we have a connection,
reject any others
"""
if self.factory.endpoint.top:
QE2LOG.warn('Qe2ClientTop.connectionMade: ' +
'rejecting second connection')
self.transport.loseConnection()
else:
QE2LOG.info('Qe2ClientTop.connectionMade: ' +
'accepting first connection')
self.factory.endpoint.top = self
self.app_connected = True
def pending_to_app(self):
"""
If there is pending data that is ready to forward to the top/app,
then send some of it along (or all of it, if there's not much).
Note that we can end up in livelock if we just send everything
through as quickly as possible. For this reason, we only send
a fixed amount through at a time.
TODO: we need a real throttling mechanism.
"""
if ((not self.top) or (not self.top.app_connected)):
QE2LOG.info('Qe2Endpoint.pending_to_app: app not connected')
return
ready_len = self.pending_in.data_ready()
if ready_len > 0:
# FIXME: this is a mistake if there's too much ready_len
# because we could blow out the transport if it can't
# keep up. What we need to do is bite off what we can
# and have a looping call that takes the rest.
#
data = self.pending_in.dequeue(ready_len)
QE2LOG.debug('Qe2Channel.pending_to_app: data %d', len(data))
# Now that we've delivered this data, we won't ask for
# it again
#
self.ack_recv += len(data)
self.top.transport.write(data)
else:
# print 'Qe2Channel.pending_to_app: NO data'
(hole_base, hole_len) = self.pending_in.first_missing()
if hole_len != -1:
QE2LOG.debug('NEED TO FILL HOLE [base %d len %d]', hole_base,
hole_len)
self.add_local_hole(hole_base, hole_base + hole_len - 1)
else:
# print 'pending_to_app: not missing anything'
pass
QE2LOG.debug('RECV THROUGH %d', self.ack_recv)
def add_remote_hole(self, new_hole_start, new_hole_end):
"""
Record that the opposite endpoint has notified us that
there is an apparent hole in their received data (by
sending us a Qe2HoleMsg).
Note that we don't just add the hole into the set verbatim;
we try to find other holes that can be combined with other
holes. (it may, in fact, be better to combine two small holes
that are separated by a non-hole in order to reduce the number
of hole-fill messages... but we don't do that right now because
the situation doesn't appear to happen in practice)
The most common case is that a hole is "extended" because
we find out that we're missing more data than we thought.
This is the most important case to handle, rather than
overlapping holes.
Holes are filled according to channel heuristics, but
recorded centrally.
"""
QE2LOG.info('add_remote_hole: adding hole (%d %d)', new_hole_start,
new_hole_end)
for (old_hole_start, old_hole_end) in sorted(self.remote_holes):
# The most common case is that a hole is "extended"
# because we find out that we're missing more data than
# we thought
#
# Another case is when a new hole fits entirely within
# an existing hole, and therefore the new hole can be ignored
#
if ((old_hole_start == new_hole_start)
and (old_hole_end < new_hole_end)):
self.remote_holes.discard((old_hole_start, old_hole_end))
self.remote_holes.add((old_hole_start, new_hole_end))
return
elif ((old_hole_start <= new_hole_start)
and (old_hole_end >= new_hole_end)):
return
self.remote_holes.add((new_hole_start, new_hole_end))
def update(self):
"""
Update the starting, running, and latent lists to reflect
the current state of each descriptor.
The assumption is that instantiated descriptors go from
starting to running to latent as they get older. Sometimes,
however, they get stuck in starting and we have to forcibly
stop them.
"""
new_latent = self.latent[:]
new_starting = list()
new_running = list()
now = time.time()
for desc in self.starting:
if desc.channel.is_connected():
desc.start_time = None
self.running.append(desc)
elif (now - desc.start_time) > self.max_starting_wait:
QE2LOG.warn('killing channel stuck in start')
desc.channel.disconnect()
new_latent.append(desc)
else:
QE2LOG.warn('channel lingering in start state')
new_starting.append(desc)
for desc in self.running:
if desc.channel.is_connected():
new_running.append(desc)
else:
QE2LOG.info('a channel has been lost')
new_latent.append(desc)
self.latent = new_latent
self.starting = new_starting
self.running = new_running
def handle_init(self, msg):
"""
Handle receipt of a OP_INIT msg.
Note that it is an error for this to be received
by a server, so QuiltServers should subclass this method
The server_quilt_uuid is the uuid assigned to this
quilt by the server. If this is the first OP_INIT
we've gotten from the server, then remember make note
of it. If it's not the first OP_INIT, then check that
the server_quilt_uuid matches previous server_quilt_uuids,
and drop the connection if it does not.
"""
server_quilt_uuid = msg.data[:16]
if self.server_quilt_uuid == None:
QE2LOG.info('got server_quilt_uuid %s',
server_quilt_uuid.encode('hex'))
self.server_quilt_uuid = server_quilt_uuid
elif self.server_quilt_uuid != server_quilt_uuid:
QE2LOG.warn('server_quilt_uuid mismatch: expected %s got %s',
self.server_quilt_uuid.encode('hex'),
server_quilt_uuid.encode('hex'))
QE2LOG.info('dropping quilt')
# If there is a channel manager, ask it to stop
# all channels
#
if self.chanman:
QE2LOG.warn('stopping all connections')
self.chanman.stop_all()
else:
QE2LOG.error('no chanman?')
# reactor.callLater(2, reactor.stop)
reactor.stop()
def disconnect(self):
"""
Disconnect with the other end of this channel
"""
QE2LOG.info('DROPPING CURVEBALL CONNECTION')
# cleanup Curveball-specific stuff
#
if self.curveball_client:
try:
QE2LOG.info('terminating curveball-client %d',
self.curveball_client.pid)
self.curveball_client.terminate()
QE2LOG.info('waiting for curveball-client')
exit_code = self.curveball_client.wait()
QE2LOG.warn('curveball-client is dead [%d]', exit_code)
except BaseException, exc:
QE2LOG.info('curveball-client did not die [%s]', str(exc))
self.curveball_client = None
def dataReceived(self, data):
QE2LOG.debug('dataReceived marker %d', self.marker)
self.recv_buf += data
(msgs, self.recv_buf) = Qe2Msg.recv(self.recv_buf)
if not msgs:
return
# The first message on a channel MUST be an OP_CHAN message
#
# NOTE: in a full-featured channel, this message will describe the
# channel and include parameters that the server should use,
# but this is not implemented. We always use the same channel
# parameters.
if not self.uuid:
first_msg = msgs[0]
if first_msg.opcode != Qe2Msg.OP_CHAN:
QE2LOG.warn('channel started/resumed without OP_CHAN msg')
self.loseConnection()
msgs = msgs[1:]
self.uuid = first_msg.uuid
listener = self.factory.server_listener
if not (self.uuid in listener.uuid2server):
QE2LOG.info('Creating Qe2Server for uuid %s',
str(self.uuid).encode('hex'))
listener.uuid2server[self.uuid] = Qe2Server(self.uuid)
self.server = listener.uuid2server[self.uuid]
# Register ourselves with our server endpoint
#
self.server.add_bottom(self)
# We should get the parameters in the first OP_CHAN message
#
# TODO: this doesn't pay attention to the first message, but
# instead makes assumptions about the parameters
#
if not self.channel:
QE2LOG.debug('CREATING CHANNEL ON SERVER')
self.channel = Qe2SocketServerChannel(self.transport,
self.server)
# Immediately tell the client the UUID we selected for our
# local UUID, so it can tell if we crash or the connections
# are redirected.
#
# TODO: it would be better if this message didn't need to
# happen immediately (because this gives the channel a
# fingerprint) but instead was based on the channel parameters.
#
# Since we're ignoring the channel paramters in this version,
# we don't have much choice, but a smarter channel would wait.
#
resp_msg = Qe2Msg(Qe2Msg.OP_INIT, self.uuid,
self.server.local_uuid, 0)
self.transport.write(resp_msg.pack())
endpoint = self.server
endpoint.process_msgs(msgs)
def pusher(self):
"""
Example of a push worker, invoked by a looping call,
to push data from the pending_out to the channel server.
This is the heart of a channel, and where the customization
for each channel takes place.
This is a very simple pusher method that can be invoked by a
LoopingCall. It is fully functional in a basic way, but is
intended to be overridden in subclasses.
The basic mechanism is to dequeue any pending data from
pending_out, and send it in a data message. If there is
no pending data, it optionally can send a chaff message,
as this example does. A pusher can send any combination
of data and chaff necessary to shape the traffic as
requested.
The pusher can also service hole requests (requests from
the opposite endpoint of the channel for missing data),
or send hole requests if any holes have been detected
locally.
The general form of a simple pusher is:
1. Return immediately if the channel is not available.
(If we're not connected yet, then don't try to send
any data)
2. If it is time for this channel to close (because it
has been open too long, or has sent the desired amount
of traffic, or some other heuristic for deciding when
a channel should be closed) then close it and return.
3. Do some combination of the following:
a) Dequeue some data from pending_out. Note the current offset.
Make a data message from the queue.
Note that when you dequeue data from pending_out, you
MUST also update two fields in the endpoint in order to
mark that the data is in the process of being sent:
self.endpoint.next_offset - the next offset to dequeue
self.endpoint.ack_send - the highest offset of any data
sent to the opposite endpoint so far
Right now, ack_send is always next_offset - 1, and it's
possible that they will be combined in the future unless
we want to get fancy with out-of-order messages.
It is typical to avoid asking for more data than can
fit in one message, but this is not required. Multiple
data messages can be composed in a single push.
b) If there is no data, or not enough data to satisfy
the traffic shaping requirements (if any) then
pad the data message with chaff and/or create one
or more chaff messages.
c) Create a hole fill request message
d) Create a data message that satisfies a hole request
As each message is created, it is added to a list.
4. Pack each Qe2Msg in the message list, and place the result
in a buffer, and write the buffer to the channel's transport.
Note that we delay packing the message list until the end
so we can get the most up-to-date metadata.
If sending more than one Qe2Msg, it can be important
to serialize them into a single write to avoid
potential synchronization issues. Doing one large write
is more efficient than several of small writes, unless small
writes are required for the channel traffic shaping (in which
case you should have dequeued less data in step #2a, or
rewritten the pusher in order to write less data in each
invocation).
"""
now = time.time()
if self.last_hole_response_time == 0:
self.last_hole_response_time = now
if self.last_hole_request_time == 0:
self.last_hole_request_time = now
# We'll always send messages with a data length of msg_data_size,
# even if we don't have any data (and need to fill it with chaff)
#
msg_data_size = self.max_msg_size
if self.should_close():
self.disconnect()
return
# If we're not connected yet, then we can't push data
# anywhere. Quietly return.
#
if not self.is_connected():
# print 'CHANNEL NOT CONNECTED YET'
return
endpoint = self.endpoint
# msgs_to_send is the list of message instances that
# we want to send. We don't pack these messages
# until we're ready to send them.
#
msgs_to_send = list()
data_msg = self.create_data_msg(0, msg_data_size)
if data_msg:
msgs_to_send.append(data_msg)
# Do we want to send more chaff?
# This is a fine place to add chaff.
# If there's a remote hole to fill, or we're already in
# the midst of filling a remote hole, add fill msgs to the list
#
# We should prefer to send fill data instead of new data because
# sending new data when there are already holes can create
# or extend holes, causing more requests.
#
"""
This doesn't seem to work yet
if not self.remote_fill_msgs:
hole = endpoint.next_remote_hole()
if hole and (not hole in self.remote_holes_filled):
self.remote_holes_filled.add(hole)
self.remote_fill_msgs = endpoint.create_hole_msg_list(
hole[0], hole[1], msg_data_size)
if self.remote_fill_msgs:
fill_res = self.remote_fill_msgs.pop()
msgs_to_send.append(fill_res)
"""
if (now - self.last_hole_response_time) > self.hole_response_interval:
fill_res = endpoint.fill_remote_hole()
if fill_res:
msgs_to_send.append(fill_res)
self.last_hole_response_time = now
QE2LOG.info('responding to a hole fill')
# Are there any holes that make us stuck, waiting to be filled?
# Ask the remote endpoint to fill them.
#
# Heuristic: don't ask for a hole request until at least
# self.hole_request_interval seconds have elapsed since the
# last request. Give the channel a chance to fill the hole
# before re-requesting
#
if (now - self.last_hole_request_time) > self.hole_request_interval:
fill_req = endpoint.request_hole_fill()
if fill_req:
# print 'PENDING SEGMENTS: %s' % str(endpoint.pending_in.segments)
msgs_to_send.append(fill_req)
self.last_hole_request_time = now
QE2LOG.info('requesting a hole fill')
# Do we want to send a ping?
# A ping will tell the endpoint our state.
#
ping_msg = self.create_ping_msg(msgs_to_send)
if ping_msg:
# print '>>>>>>>>>>> SENDING PING'
msgs_to_send.append(ping_msg)
# If we created any messages, then pack them. If the result
# is a non-empty string of packed data, then write it to the
# transport.
#
packed = ''
for msg in msgs_to_send:
msg.ack_send = endpoint.ack_send
msg.ack_recv = endpoint.ack_recv
QE2LOG.debug('sending %s', str(msg))
packed += msg.pack()
if packed:
# If we're sending anything, then update self.next_ping_time
#
self.next_ping_time = time.time() + self.max_idle_time
transport = self.get_transport()
transport.write(packed)
QE2LOG.debug('ack_send is %d', endpoint.ack_send)
def process_msgs(self, msgs):
"""
Process messages received by a channel
"""
max_offset = -1
delivered = False
old_remote_ack_recv = self.remote_ack_recv
for msg in msgs:
QE2LOG.debug('RECEIVE MSG %s', str(msg))
# update max_offset and remote_ack_recv, no
# matter what the message type is
#
if max_offset < msg.ack_send:
max_offset = msg.ack_send
if self.remote_ack_recv < msg.ack_recv:
self.remote_ack_recv = msg.ack_recv
if self.remote_ack_send < msg.ack_send:
self.remote_ack_send = msg.ack_send
if msg.opcode == Qe2Msg.OP_DATA:
if len(msg.data) > 0:
self.deliver(msg.send_offset, msg.data)
delivered = True
elif msg.opcode == Qe2Msg.OP_PING:
pass
elif msg.opcode == Qe2Msg.OP_HOLE:
if self.remote_ack_recv > msg.hole[0]:
QE2LOG.info('UNEXPECTED hole start before remote_ack_recv')
self.add_remote_hole(msg.hole[0],
msg.hole[0] + msg.hole[1] - 1)
elif msg.opcode == Qe2Msg.OP_CHAN:
QE2LOG.info('got OP_CHAN message')
pass
elif msg.opcode == Qe2Msg.OP_HALT:
# TODO: this should close down the quilt (not just
# one particular channel)
#
QE2LOG.info('got OP_HALT message')
elif msg.opcode == Qe2Msg.OP_INIT:
self.handle_init(msg)
else:
QE2LOG.error('UNHANDLED msg %d', msg.opcode)
# If this sequence of msgs included delivered data, then
# see if there's anything to push to the app
#
if delivered:
self.pending_to_app()
if max_offset > self.ack_recv:
QE2LOG.info('max_offset %d > self.ack_rev %d: something missing',
max_offset, self.ack_recv)
self.add_local_hole(self.ack_recv + 1, max_offset)
QE2LOG.debug('LOCAL holes %s', str(sorted(self.local_holes)))
# We've gotten acknowledgment from the remote endpoint
# for additional data. Throw away our old copy.
#
# TODO: it is much more efficient to delete larger chunks
# periodically rather than small chunks constantly.
#
if old_remote_ack_recv < self.remote_ack_recv:
self.pending_out.discard(self.remote_ack_recv -
old_remote_ack_recv)
def connectionMade(self):
QE2LOG.info('Qe2ServerBottom.connectionMade()')
self.factory.server_bottom = self