def main(local_ip, mca):
my_sock = UdpSocket((local_ip, 0))
my_sock.receive_buffer_size = 8000
my_sock.send_buffer_size = 100000
mc_sock = UdpSocket((mca, 8000))
mc_sock.join_mcast_group(mca)
mc_sock.receive_buffer_size = 8000
mc_sock.send_buffer_size = 100000
receiver = Receiver()
handler = RmcProtocolHandler(my_sock, mc_sock)
handler.MAX_BACKLOG = 1000
handler.MAX_BURST = 10
## handler.add_action('__received_packet__', receiver.on_incomming_packet)
## handler.add_action('__sent_packet__', receiver.on_outgoing_packet)
handler.add_action('missing_heartbeat', receiver.on_missing_sender)
handler.add_action('new_sender', receiver.on_new_sender)
handler.add_action('new_packet', receiver.on_new_packet)
handler.add_action('got_heartbeat', receiver.on_heartbeat)
handler.add_action('got_reset', receiver.on_reset)
handler.add_action('got_lost', receiver.on_lost)
handler.add_action('got_nack', receiver.on_nack)
handler.add_action('sent_heartbeat', receiver.sent_heartbeat)
handler.add_action('sent_lost', receiver.sent_lost)
handler.add_action('sent_nack', receiver.sent_nack)
print 'RECEIVER STARTED', repr(handler.local_addr)
try:
event.dispatch()
except KeyboardInterrupt:
pass
handler.finish()
now = time.time()
print '%.6f: DONE' % now
return
def activate(self):
"""Enable receive via multicast.
Joins multicast group and creates reactor.
"""
if self.is_active:
return
self.mc_socket = UdpSocket()
self.mc_socket.bind(self.dest_addr)
self.mc_socket.multicast_interface = self.dest_if
self.mc_socket.join_mcast_group(self.dest_addr[0])
self.mc_reactor = RmcReactor(self.mc_socket, self)
self.mc_reactor.add_read()
self._hb_timer = event.timeout(0.1, self._hb_timer_tick)
self._chk_timer = event.timeout(0.01, self._chk_timer_tick)
self.is_active = True
return
class RmcProtocolHandler(object):
"""Class implementing the RMC protocol.
The protocol is based on negative acknowledge (NACK) only sent
when a receiver sees a gap in the sequence numbers of a packet
stream (from a sender).
If packets are received in sequence, no acknowledge is sent; the
packets are consumed silently. If a receiver detects that one or
more packets are missing, it starts to send NACK packets directly
to the sender. The sender then tries to fullfill the requests from
his packet backlog.
After sending a data (aka non-protocol) packet, heartbeat packets
are sent. The heartbeat packets do not contain any data, they carry
only the last used sequence no. of this sender. The rate the
heartbeat packets are sent is variable, meaning that the time
between heartbeats will increase over time. This allows for fast
synchronisation after new packets have been sent and at the same
time reduces overall traffic.
For the following events listners (observers etc.) can be
registered via add_action:
new_sender:
emitted when a packet from an unknown address has
been received.
Param: sender
missing_heartbeat:
emitted when this process failed to receive an
expected heartbeat from a known sender.
Param: sender
got_heartbeat:
emitted when this process received a heartbeat packet.
Param: packet
got_reset:
emitted when a RESET packet has been received.
Param: packet
got_lost:
emitted when a LOST packet has been received
Param: packet
got_nack:
emitted when a NACK packet has been received
Param: packet
packet_sent:
emitted after a data packet has been sent out
Param: packet
sent_heartbeat:
emitted after the heartbeat packet has been spooled
Param: packet
sent_nack:
emitted after a NACK packet has been spooled
Param: packet
sent_lost:
emitted after a LOST packet has been spooled
Param: packet
new_packet:
emitted when a new data packet arrived
Param: packet
sequence_overflow:
emitted when seq becomes > 2.000.000.000
Param: None
__received_packet__:
emitted on every packet received (for debugging!)
Param: packet
__sent_packet__:
emitted on every packet sent (for debugging!)
Param: packet
The callback always receives 2 parameters: the first one is shown in the list above as Param,
the 2nd one is always the RmcProtocolHandler instance.
"""
# max. no. of packets to send from the queue on one write event.
MAX_BURST = 10
# list of times to wait between heartbeats
_def_times = (0.01, 0.02, 0.05, 0.05, 0.075, 0.1, 0.2, 0.25, 0.5) #, 1.0, 2.0, 5.0)
# max. length of backlog for sent packets
MAX_BACKLOG = 100
# valid event names
EVENTS = sorted(['packet_sent', 'sent_heartbeat', 'sent_lost', 'sent_nack',
'got_nack', 'got_lost', 'got_reset', 'got_heartbeat',
'new_sender', 'missing_heartbeat',
'new_packet',
# these are for debugging
'__received_packet__', '__sent_packet__'])
def __init__(self, uc_socket, mc_socket=None):
"""Setup rmc handler.
uc_socket: UdpSocket bound to local unicast address
mc_socket: UdpSocket bound to multicast address, optional
Attributes:
mc_socket: UdpSocket bound to multicast address
uc_socket: UdpSocket bound to local unicast address
mc_reactor: RmcReactor for multicast address (receive only)
uc_reactor: RmcReactor for unicast address (receive/send)
dest_addr: ip/port of multicast channel
local_addr: ip/port for unicast
seq: current sequence no. for outgoing packets
no_sync: if True don't send NACKs on first heartbeat for a sender
silent : if True don't send data, never ever, just listen
"""
# create multicast socket
## self.mc_socket = UdpSocket((mc_addr[0], mc_addr[1]))
## self.mc_socket.multicast_interface = mc_interface
## self.mc_socket.join_mcast_group(mc_addr[0])
self.mc_socket = mc_socket
self.uc_socket = uc_socket
# create reactors for both sockets
# and attach them to this protocol handler
self.uc_reactor = RmcReactor(self.uc_socket, self)
self.local_addr = self.uc_socket.getsockname()
# if multicast socket is None
# sending will use the unicast socket as fallback
if mc_socket is None:
self.mc_reactor = self.uc_reactor
## self.mc_socket.multicast_interface = mc_interface
else:
self.mc_reactor = RmcReactor(mc_socket, self)
self.mc_socket.multicast_loop = True
# get/assign addresses
self.dest_addr = self.mc_socket.getsockname()
# send_buffer is a list of packets to send out
# send_prio_buffer is for outgoing packets w/ high priority
self._send_buffer = []
self._send_prio_buffer = []
# sequence counter for outgoing data packets
self.seq = 0
# remember last data packet sent
self._last_sent = None
# keep track of peers
self._peers = {}
# our outgoing backlog
self._backlog = []
# activate reactors
self.mc_reactor.add_read()
self.uc_reactor.add_read()
# initialize generator for hearbeat times
self._hb_gen = self._hb_generator()
self._hb_gen.next()
# mapping to keep track of event subscriptions
self._ev_handler = {}
# flag if write event is active
self._write_active = False
# flag when set, no sync on first heartbeat of a sender is done
self.no_sync = False
# flag when set, no data will be sent (ever!)
self._silent = False
# prepare continous timer ticks
self._hb_timer = event.timeout(0.1, self._hb_timer_tick)
self._chk_timer = event.timeout(0.01, self._chk_timer_tick)
# flag showing if this handler does receive data via multicast
self.is_active = True
return
def _get_silent(self):
return self._silent
def _set_silent(self, value):
if value:
self._silent = True
if not self.no_sync:
self.no_sync = True
else:
self._silent = False
return
# when set, no data will be sent
# will set no_sync to True if set to True
silent = property(_get_silent, _set_silent)
def finish(self):
"""Deregister and close sockets and clean-up.
"""
self.silent = True
self.deactivate()
self.uc_reactor.close()
self.uc_reactor = None
self._ev_handler = {}
self.seq = -1
self._send_prio_buffer = []
self._send_buffer = []
return
def deactivate(self):
"""Disable receive of data via mutlicast.
Leaves the multicast group and destroys the mc reactor.
"""
if not self.is_active:
return
self._hb_timer.delete()
self._hb_timer = None
self._chk_timer.delete()
self._chk_timer = None
self.mc_socket.leave_mcast_group(self.dest_addr[0])
self.mc_reactor.close()
self.mc_socket.close()
self.mc_reactor = None
self.mc_socket = None
self.is_active = False
return
def activate(self):
"""Enable receive via multicast.
Joins multicast group and creates reactor.
"""
if self.is_active:
return
self.mc_socket = UdpSocket()
self.mc_socket.bind(self.dest_addr)
self.mc_socket.multicast_interface = self.dest_if
self.mc_socket.join_mcast_group(self.dest_addr[0])
self.mc_reactor = RmcReactor(self.mc_socket, self)
self.mc_reactor.add_read()
self._hb_timer = event.timeout(0.1, self._hb_timer_tick)
self._chk_timer = event.timeout(0.01, self._chk_timer_tick)
self.is_active = True
return
def set_unicast_socket(self, uc_socket, mc_interface=''):
"""Set the unicast socket to the given one.
Will create a new RmcReactor for this socket and attach it to
this instance.
If uc_socket is None, the multicast socket is set as fallback.
Will update self.local_addr to reflect the new setting.
"""
self.uc_socket = uc_socket
if uc_socket:
self.uc_reactor = RmcReactor(uc_socket, self)
self.uc_socket.multicast_interface = mc_interface
self.uc_socket.multicast_loop = True
else:
self.uc_reactor = self.mc_reactor
self.local_addr = self.uc_socket.getsockname()
return
def add_action(self, event, action):
"""Add the callable action to the given event.
action needs to accept 2 parameters:
1. param (packet, sender etc., depends on event)
2. a RmcProtocolHandler instance
"""
event = event.strip().lower()
if event not in self.EVENTS:
raise ValueError('unknown event', event)
handlers = self._ev_handler.setdefault(event, [])
handlers.append(action)
return
def remove_action(self, event, action):
"""Remove the given action from event.
"""
event = event.strip().lower()
if event not in self.EVENTS:
raise ValueError('unknown event', event)
handlers = self._ev_handler.setdefault(event, [])
if handlers and action in handlers:
handlers.remove(action)
return
def abort_action(self):
"""Raise AbortHandling exception.
"""
raise AbortHandling()
def get_peer(self, addr):
"""Return Peer instance for given address or None.
If no peer is known for the given address, None is returned.
"""
ret = self._peers.get(addr, None)
return ret
def del_peer(self, addr):
"""Remove the peer with the given address.
Do nothing if peer is not known.
"""
try:
del self._peers[addr]
except KeyError:
pass
return
def set_heartbeat_times(self, hb_times, hb_index=0):
"""Set the heartbeat delay times to the given sequence.
hb_times: sequence of floats specifying the delay between heartbeats.
hb_index: new index into given heartbeats.
Will also send the HeartbeatTimesPacket w/ the given times and index.
"""
self._hb_times = tuple(hb_times)
self._hb_index = hb_index
p = HeartbeatTimesPacket(hb_index, hb_times)
self.send(p)
return
def _notify_handlers(self, event, param):
"""Helper calling all handlers for event with the given parameter.
"""
handlers = self._ev_handler.get(event)
if handlers:
try:
for action in handlers:
action(param, self)
except AbortHandling:
# either self.abort_action() has been called
# or AbortHandling has been raised inside a
# notification handler
pass
return
def send(self, pkt, dest_addr=None):
"""Send data packet.
Will add the packet to _send_buffer and activate write events.
If dest_addr is not set, the mulicast address is used.
Adds destination address to packet and sets sequence no.
"""
if self.silent:
# never ever send data
return
if not dest_addr:
# if no destination address given, use multicast channel
dest_addr = self.dest_addr
if not (pkt.is_protocol or pkt.is_resent):
# plain data packet, set seq
if pkt.seq < 0:
pkt.seq = self.seq
self.seq += 1
elif self.seq <= pkt.seq:
# increase last sent seq
self.seq = pkt.seq + 1
else:
# can't decrease seq
raise InvalidPacketSequence('invalid sequence: %d' % pkt.seq)
if pkt.flags & packet.RESET:
self.seq = pkt.seq + 1
# clean backlog on RESET
bl = [ p for p in self._backlog if p.seq >= pkt.seq ]
self._backlog = bl
self._last_sent = pkt
# attach destination address
pkt.dest_addr = dest_addr
if pkt.is_protocol or pkt.is_resent:
# protocol messages and resent packets have higher priority
self._send_prio_buffer.append(pkt)
else:
self._send_buffer.append(pkt)
# sending occurs on the next write event
if not self._write_active:
self._write_active = True
self.uc_reactor.add_write()
return
def _packet_sent(self, pkt):
"""Helper method called after the given packet has been sent.
Shrinks backlog if necessary then adds packet to backlog and
(re-)sets the heartbeat timer.
"""
# shrink backlog
l_diff = len(self._backlog) - self.MAX_BACKLOG
if l_diff > 0:
self._backlog = self._backlog[l_diff:]
if not pkt.is_resent:
# mark passed packet as last sent packet for heartbeat
# call event listners and reset heartbeat time
self._last_sent = pkt
self._backlog.append(pkt)
self._notify_handlers('packet_sent', pkt)
self.reset_heartbeat()
# check for seq overflow
if self.seq > 2000000000:
# emit sequence_overflow
self._notify_handlers('sequence_overflow', None)
return
def _send_packets(self, packets):
uc_send = self.uc_reactor.send
notify = self._notify_handlers
p_sent = self._packet_sent
for p in packets:
uc_send(str(p), p.dest_addr)
notify('__sent_packet__', p)
if not (p.is_protocol or p.is_resent):
# new normal packet, post-process it
p_sent(p)
return
def on_write(self):
"""Event handler called when reactor can send more data.
Will send at most MAX_BURST packets at once.
"""
# first send all priority packets collected
out_list = self._send_prio_buffer[:self.MAX_BURST]
self._send_prio_buffer = self._send_prio_buffer[self.MAX_BURST:]
out_len = len(out_list)
if out_list:
self._send_packets(out_list)
# try to send normal packets
if out_len < self.MAX_BURST:
r = self.MAX_BURST - out_len
out_list = self._send_buffer[:r]
self._send_buffer = self._send_buffer[r:]
self._send_packets(out_list)
if self._send_prio_buffer or self._send_buffer:
# more data to send, re-activate write events
self._write_active = True
self.uc_reactor.add_write()
else:
# nothing to do, disable write events
self._write_active = False
self.uc_reactor.del_write()
return
def _get_oldest_sender(self):
"""Helper returning the sender with the smallest
last_heartbeat value or None.
"""
if not self._peers:
return None
lp = operator.attrgetter('last_packet')
senders = sorted(self._peers.values(), key=lp)
return senders[0]
def _hb_generator(self):
"""Helper method that creates a generator for the heartbeat
delay interval.
Uses yield expression!
"""
try:
self._hb_times
except AttributeError:
self._hb_times = self._def_times
self._hb_index = 0
while True:
times = self._hb_times
try:
t_next = times[self._hb_index]
except IndexError:
t_next = times[-1]
self._hb_index = len(times) - 1
## t_next = rnd_delay_time(t_next)
ret = (yield t_next)
if ret:
self._hb_index = 0
elif self._hb_index >= 0:
self._hb_index += 1
return
def _next_hb_time(self, reset=None):
"""Helper method returning the delay time for the next heartbeat.
If reset is set, the heartbeat delay is reset to the smallest
value.
Uses .send method for generators!
"""
ret = self._hb_gen.send(reset)
return ret
def _check_sender_timeouts(self):
"""Helper method to check for senders who timed out.
Notify 'missing_heartbeat' listners of senders who's last package
was received more then their current heartbeat delay in the past.
"""
now = time.time()
pv = self._peers.values()
timed_out = [s for s in pv if s.heartbeat_overdue(now)]
# notify listners of missing senders
# the action handler is responsible for removing the sender, if wanted
for sender in timed_out:
self._notify_handlers('missing_heartbeat', sender)
return
def on_heartbeat(self):
"""Event handler called from heartbeat timer.
Send next heartbeat if needed and adjust timer interval.
"""
self._check_sender_timeouts()
if self.seq < 0 or self._send_buffer:
# nothing sent so far
t_next = 0.01
elif not self._last_sent:
t_next = 0.01
elif self.seq == (self._last_sent.seq + 1):
# no change, use next HB interval
# send HB packet
p_last = self._last_sent
seq = p_last.seq
# adjust timer
t_next = self._next_hb_time()
p = HeartbeatPacket(seq, heartbeat=self._hb_index)
self.send(p)
self._notify_handlers('sent_heartbeat', p)
else:
# something changed, reset heartbeat
t_next = self._next_hb_time(True)
return t_next
def _hb_timer_tick(self):
self._hb_timer.delete()
t_next = self.on_heartbeat()
self._hb_timer = event.timeout(t_next, self._hb_timer_tick)
return
def _chk_timer_tick(self):
self._chk_timer.delete()
self._check_sender_timeouts()
self._chk_timer = event.timeout(0.01, self._chk_timer_tick)
return
def reset_heartbeat(self):
"""Reset heartbeat time to lowest value.
"""
if not self._hb_timer:
return
self._hb_timer.delete()
t_next = self._next_hb_time(True)
self._hb_timer = event.timeout(t_next, self._hb_timer_tick)
return t_next
def _deliver_look_ahead(self, sender, start=None):
"""Helper to deliver packets from this look-ahead cache that
are in sequence starting at start.
If start is None, sender.seq+1 is assumed.
Returns last delivered seq or None if nothing was sent.
"""
in_seq = sender.get_look_ahead(start)
if not in_seq:
# nothing found
return None
last_seen = in_seq[-1].seq
# deliver packets
for p in in_seq:
self._notify_handlers('new_packet', p)
# purge cache
sender.purge_look_ahead(last_seen)
return last_seen
def _send_nack_range(self, sender, count=1, now=None):
"""Helper to send out NACK packets.
"""
if now is None:
now = time.time()
sent_nacks = sender.sent_nacks
n_seq = sender.seq + 1
while count > 0:
if n_seq in sent_nacks:
# check for nack resend
t_nack, t_first = sent_nacks[n_seq]
if (now - t_first) >= sender.nack_timeout:
# old NACK, drop it
del sent_nacks[n_seq]
elif (now - t_nack) >= sender.nack_resend_time:
# time to re-send NACK
p = NackPacket(n_seq)
self.send(p, sender.address)
self._notify_handlers('sent_nack', p)
sent_nacks[n_seq] = (now, t_first)
elif len(sent_nacks) < sender.max_nacks:
# new NACK to send
p = NackPacket(n_seq)
self.send(p, sender.address)
self._notify_handlers('sent_nack', p)
sent_nacks[n_seq] = (now, now)
# next in sequence
n_seq += 1
count -= 1
return
def _deal_with_nack(self, pkt):
"""Helper method to deal with received NACK packet.
If the requested packet is in the backlog, it's sent to the
address the nack has been received from.
If not a LOST packet is sent.
"""
sender = pkt.sender
# mark sender as still alive
sender.touch()
# notify interested parties
self._notify_handlers('got_nack', pkt)
# check for LOST condition
addr = pkt.src_addr
sent = False
if not self._backlog:
# no backlog
# send RESET packet
p = packet.Packet(self.seq)
p.flags |= packet.RESET
self.send(p, addr)
self._notify_handlers('sent_lost', p)
return
p0 = self._backlog[0]
if pkt.seq < p0.seq:
# if the requested seq is smaller than the smallest seq in
# our backlog send a LOST packet with the lowest available
# seq to cut down further nacks
seq = p0.seq
p = LostPacket(seq)
# targeted to addr, others wont see it!
self.send(p, addr)
self._notify_handlers('sent_lost', p)
return
# packet still in backlog
# try to send it
for p in self._backlog:
if pkt.seq == p.seq:
# re-send requested packets
p.flags |= packet.RESENT
# unicast send!
self.send(p, addr)
sent = True
break
if not sent:
# snafu
# packet not in backlog anymore
# send LostPacket
p0 = self._backlog[0]
p = LostPacket(p0.seq)
self.send(p, addr)
self._notify_handlers('sent_lost', p)
else:
# speed up recovery
self.reset_heartbeat()
return
def _deal_with_lost(self, pkt):
"""Helper method to deal with a received LOST packet.
Clean up expected packet sequences (aka sent nacks) and shrink
the look-ahead cache. Then try to deliver packets from the
look-ahead cache.
"""
sender = pkt.sender
# packets with a seq lower then the given can't be recovered
# remove them from outstanding nack list and adjust last_seen
sender.purge_sent_nacks(pkt.seq)
sender.purge_look_ahead(pkt.seq)
# notify listners before delivery of data packets
sender.touch(pkt.seq - 1)
self._notify_handlers('got_lost', pkt)
# send nack for smallest available packet
self._send_nack_range(sender)
return
def _deal_with_reset(self, pkt):
"""Helper method to deal with a received RESET packet.
Will reset (aka clear) all data for this sender and set
senders seq to the one in the packet.
"""
sender = pkt.sender
# forget all information on sender
sender.reset_all(pkt.seq)
self._notify_handlers('got_reset', pkt)
return
def _deal_with_heartbeat(self, pkt):
"""Helper method to deal with a received HEARTBEAT packet.
"""
sender = pkt.sender
self._notify_handlers('got_heartbeat', pkt)
now = time.time()
last_seen = sender.seq
if last_seen < 0 and self.no_sync:
# avoid sending nack on first heartbeat of sender when no_sync is set
sender.touch(pkt.seq, heartbeat=pkt.heartbeat)
return
sender.touch(heartbeat=pkt.heartbeat, now=now)
pdiff = pkt.seq - last_seen
if pdiff > 0:
# we missed at least one packet
# send NACKs
self._send_nack_range(sender, pdiff, now)
return
def _deal_with_hb_times(self, pkt):
"""Helper method to deal with a received HB_TIMES packet.
"""
sender = pkt.sender
now = time.time()
times = packet.decode_times(pkt)
sender.set_heartbeat_times(times)
sender.touch(heartbeat=pkt.heartbeat, now=now)
return
def _protocol_packet(self, pkt):
"""Handle protocol specific packet received from addr.
"""
if pkt.flags & packet.NACK:
self._deal_with_nack(pkt)
elif pkt.flags & packet.LOST:
self._deal_with_lost(pkt)
elif pkt.flags & packet.HEARTBEAT:
self._deal_with_heartbeat(pkt)
elif pkt.flags & packet.HB_TIMES:
self._deal_with_hb_times(pkt)
elif pkt.flags & packet.RESET:
self._deal_with_reset(pkt)
elif pkt.flags & packet.ACK:
# shouldn't we be happy?
# sure, but we ignore ACK packets
pass
return
def _data_packet(self, pkt):
"""Deal with ordinary data packet.
Checks for and deals with missing packets.
"""
sender = pkt.sender
if sender.seq < 0 and self.no_sync:
# don't send nacks on first packet
sender.seq = pkt.seq - 1
now = time.time()
pdiff = pkt.seq - sender.seq
sent_nacks = sender.sent_nacks
look_ahead = sender.look_ahead
last_seen = sender.seq
if pdiff == 1:
# received next packet in sequence
# deal with it
if pkt.seq in sent_nacks:
del sent_nacks[pkt.seq]
self._notify_handlers('new_packet', pkt)
sender.seq = pkt.seq
# try to deliver cached packets
last_seen = self._deliver_look_ahead(sender, pkt.seq + 1)
sender.touch(seq=last_seen, heartbeat=pkt.heartbeat)
elif pdiff > 1:
# we missed at least one packet
# forward caching of packets
# reduces the number of nacks needed
look_ahead[pkt.seq] = pkt
# sent NACKs for missing packets
self._send_nack_range(sender, pdiff, now)
sender.touch(heartbeat=pkt.heartbeat)
else:
# ouch...
# but we mark sender as still alive
sender.touch(heartbeat=pkt.heartbeat)
return
def on_packet(self, p):
"""Event handler called when reactor received a new rmc packet.
p is the packet
"""
if p.src_addr == self.local_addr:
# we get our own packets due to the multicast mechanism
# ignore them
return
if p.recv_addr == self.local_addr:
# unicast aka private
p.flags |= packet.PRIVATE
# fetch/create Peer instance for packet sender
sender = self._peers.get(p.src_addr, None)
if not sender:
# new (unknown) sender notification
sender = Peer(p.src_addr, times=self._def_times)
self._peers[p.src_addr] = sender
# new sender, notify listners
self._notify_handlers('new_sender', sender)
p.sender = sender
# call event handler for *all* packets, debugging only
# otherwise it'll be slow
self._notify_handlers('__received_packet__', p)
# dispatch packet
if p.flags & packet.PROTOFLAGS:
self._protocol_packet(p)
else:
self._data_packet(p)
return