def activate(self): """Enable receive via multicast. Joins multicast group and creates reactor. """ if self.is_active: return self.mc_socket = UdpSocket() self.mc_socket.bind(self.dest_addr) self.mc_socket.multicast_interface = self.dest_if self.mc_socket.join_mcast_group(self.dest_addr[0]) self.mc_reactor = RmcReactor(self.mc_socket, self) self.mc_reactor.add_read() self._hb_timer = event.timeout(0.1, self._hb_timer_tick) self._chk_timer = event.timeout(0.01, self._chk_timer_tick) self.is_active = True return
def set_unicast_socket(self, uc_socket, mc_interface=''): """Set the unicast socket to the given one. Will create a new RmcReactor for this socket and attach it to this instance. If uc_socket is None, the multicast socket is set as fallback. Will update self.local_addr to reflect the new setting. """ self.uc_socket = uc_socket if uc_socket: self.uc_reactor = RmcReactor(uc_socket, self) self.uc_socket.multicast_interface = mc_interface self.uc_socket.multicast_loop = True else: self.uc_reactor = self.mc_reactor self.local_addr = self.uc_socket.getsockname() return
class RmcProtocolHandler(object): """Class implementing the RMC protocol. The protocol is based on negative acknowledge (NACK) only sent when a receiver sees a gap in the sequence numbers of a packet stream (from a sender). If packets are received in sequence, no acknowledge is sent; the packets are consumed silently. If a receiver detects that one or more packets are missing, it starts to send NACK packets directly to the sender. The sender then tries to fullfill the requests from his packet backlog. After sending a data (aka non-protocol) packet, heartbeat packets are sent. The heartbeat packets do not contain any data, they carry only the last used sequence no. of this sender. The rate the heartbeat packets are sent is variable, meaning that the time between heartbeats will increase over time. This allows for fast synchronisation after new packets have been sent and at the same time reduces overall traffic. For the following events listners (observers etc.) can be registered via add_action: new_sender: emitted when a packet from an unknown address has been received. Param: sender missing_heartbeat: emitted when this process failed to receive an expected heartbeat from a known sender. Param: sender got_heartbeat: emitted when this process received a heartbeat packet. Param: packet got_reset: emitted when a RESET packet has been received. Param: packet got_lost: emitted when a LOST packet has been received Param: packet got_nack: emitted when a NACK packet has been received Param: packet packet_sent: emitted after a data packet has been sent out Param: packet sent_heartbeat: emitted after the heartbeat packet has been spooled Param: packet sent_nack: emitted after a NACK packet has been spooled Param: packet sent_lost: emitted after a LOST packet has been spooled Param: packet new_packet: emitted when a new data packet arrived Param: packet sequence_overflow: emitted when seq becomes > 2.000.000.000 Param: None __received_packet__: emitted on every packet received (for debugging!) Param: packet __sent_packet__: emitted on every packet sent (for debugging!) Param: packet The callback always receives 2 parameters: the first one is shown in the list above as Param, the 2nd one is always the RmcProtocolHandler instance. """ # max. no. of packets to send from the queue on one write event. MAX_BURST = 10 # list of times to wait between heartbeats _def_times = (0.01, 0.02, 0.05, 0.05, 0.075, 0.1, 0.2, 0.25, 0.5) #, 1.0, 2.0, 5.0) # max. length of backlog for sent packets MAX_BACKLOG = 100 # valid event names EVENTS = sorted(['packet_sent', 'sent_heartbeat', 'sent_lost', 'sent_nack', 'got_nack', 'got_lost', 'got_reset', 'got_heartbeat', 'new_sender', 'missing_heartbeat', 'new_packet', # these are for debugging '__received_packet__', '__sent_packet__']) def __init__(self, uc_socket, mc_socket=None): """Setup rmc handler. uc_socket: UdpSocket bound to local unicast address mc_socket: UdpSocket bound to multicast address, optional Attributes: mc_socket: UdpSocket bound to multicast address uc_socket: UdpSocket bound to local unicast address mc_reactor: RmcReactor for multicast address (receive only) uc_reactor: RmcReactor for unicast address (receive/send) dest_addr: ip/port of multicast channel local_addr: ip/port for unicast seq: current sequence no. for outgoing packets no_sync: if True don't send NACKs on first heartbeat for a sender silent : if True don't send data, never ever, just listen """ # create multicast socket ## self.mc_socket = UdpSocket((mc_addr[0], mc_addr[1])) ## self.mc_socket.multicast_interface = mc_interface ## self.mc_socket.join_mcast_group(mc_addr[0]) self.mc_socket = mc_socket self.uc_socket = uc_socket # create reactors for both sockets # and attach them to this protocol handler self.uc_reactor = RmcReactor(self.uc_socket, self) self.local_addr = self.uc_socket.getsockname() # if multicast socket is None # sending will use the unicast socket as fallback if mc_socket is None: self.mc_reactor = self.uc_reactor ## self.mc_socket.multicast_interface = mc_interface else: self.mc_reactor = RmcReactor(mc_socket, self) self.mc_socket.multicast_loop = True # get/assign addresses self.dest_addr = self.mc_socket.getsockname() # send_buffer is a list of packets to send out # send_prio_buffer is for outgoing packets w/ high priority self._send_buffer = [] self._send_prio_buffer = [] # sequence counter for outgoing data packets self.seq = 0 # remember last data packet sent self._last_sent = None # keep track of peers self._peers = {} # our outgoing backlog self._backlog = [] # activate reactors self.mc_reactor.add_read() self.uc_reactor.add_read() # initialize generator for hearbeat times self._hb_gen = self._hb_generator() self._hb_gen.next() # mapping to keep track of event subscriptions self._ev_handler = {} # flag if write event is active self._write_active = False # flag when set, no sync on first heartbeat of a sender is done self.no_sync = False # flag when set, no data will be sent (ever!) self._silent = False # prepare continous timer ticks self._hb_timer = event.timeout(0.1, self._hb_timer_tick) self._chk_timer = event.timeout(0.01, self._chk_timer_tick) # flag showing if this handler does receive data via multicast self.is_active = True return def _get_silent(self): return self._silent def _set_silent(self, value): if value: self._silent = True if not self.no_sync: self.no_sync = True else: self._silent = False return # when set, no data will be sent # will set no_sync to True if set to True silent = property(_get_silent, _set_silent) def finish(self): """Deregister and close sockets and clean-up. """ self.silent = True self.deactivate() self.uc_reactor.close() self.uc_reactor = None self._ev_handler = {} self.seq = -1 self._send_prio_buffer = [] self._send_buffer = [] return def deactivate(self): """Disable receive of data via mutlicast. Leaves the multicast group and destroys the mc reactor. """ if not self.is_active: return self._hb_timer.delete() self._hb_timer = None self._chk_timer.delete() self._chk_timer = None self.mc_socket.leave_mcast_group(self.dest_addr[0]) self.mc_reactor.close() self.mc_socket.close() self.mc_reactor = None self.mc_socket = None self.is_active = False return def activate(self): """Enable receive via multicast. Joins multicast group and creates reactor. """ if self.is_active: return self.mc_socket = UdpSocket() self.mc_socket.bind(self.dest_addr) self.mc_socket.multicast_interface = self.dest_if self.mc_socket.join_mcast_group(self.dest_addr[0]) self.mc_reactor = RmcReactor(self.mc_socket, self) self.mc_reactor.add_read() self._hb_timer = event.timeout(0.1, self._hb_timer_tick) self._chk_timer = event.timeout(0.01, self._chk_timer_tick) self.is_active = True return def set_unicast_socket(self, uc_socket, mc_interface=''): """Set the unicast socket to the given one. Will create a new RmcReactor for this socket and attach it to this instance. If uc_socket is None, the multicast socket is set as fallback. Will update self.local_addr to reflect the new setting. """ self.uc_socket = uc_socket if uc_socket: self.uc_reactor = RmcReactor(uc_socket, self) self.uc_socket.multicast_interface = mc_interface self.uc_socket.multicast_loop = True else: self.uc_reactor = self.mc_reactor self.local_addr = self.uc_socket.getsockname() return def add_action(self, event, action): """Add the callable action to the given event. action needs to accept 2 parameters: 1. param (packet, sender etc., depends on event) 2. a RmcProtocolHandler instance """ event = event.strip().lower() if event not in self.EVENTS: raise ValueError('unknown event', event) handlers = self._ev_handler.setdefault(event, []) handlers.append(action) return def remove_action(self, event, action): """Remove the given action from event. """ event = event.strip().lower() if event not in self.EVENTS: raise ValueError('unknown event', event) handlers = self._ev_handler.setdefault(event, []) if handlers and action in handlers: handlers.remove(action) return def abort_action(self): """Raise AbortHandling exception. """ raise AbortHandling() def get_peer(self, addr): """Return Peer instance for given address or None. If no peer is known for the given address, None is returned. """ ret = self._peers.get(addr, None) return ret def del_peer(self, addr): """Remove the peer with the given address. Do nothing if peer is not known. """ try: del self._peers[addr] except KeyError: pass return def set_heartbeat_times(self, hb_times, hb_index=0): """Set the heartbeat delay times to the given sequence. hb_times: sequence of floats specifying the delay between heartbeats. hb_index: new index into given heartbeats. Will also send the HeartbeatTimesPacket w/ the given times and index. """ self._hb_times = tuple(hb_times) self._hb_index = hb_index p = HeartbeatTimesPacket(hb_index, hb_times) self.send(p) return def _notify_handlers(self, event, param): """Helper calling all handlers for event with the given parameter. """ handlers = self._ev_handler.get(event) if handlers: try: for action in handlers: action(param, self) except AbortHandling: # either self.abort_action() has been called # or AbortHandling has been raised inside a # notification handler pass return def send(self, pkt, dest_addr=None): """Send data packet. Will add the packet to _send_buffer and activate write events. If dest_addr is not set, the mulicast address is used. Adds destination address to packet and sets sequence no. """ if self.silent: # never ever send data return if not dest_addr: # if no destination address given, use multicast channel dest_addr = self.dest_addr if not (pkt.is_protocol or pkt.is_resent): # plain data packet, set seq if pkt.seq < 0: pkt.seq = self.seq self.seq += 1 elif self.seq <= pkt.seq: # increase last sent seq self.seq = pkt.seq + 1 else: # can't decrease seq raise InvalidPacketSequence('invalid sequence: %d' % pkt.seq) if pkt.flags & packet.RESET: self.seq = pkt.seq + 1 # clean backlog on RESET bl = [ p for p in self._backlog if p.seq >= pkt.seq ] self._backlog = bl self._last_sent = pkt # attach destination address pkt.dest_addr = dest_addr if pkt.is_protocol or pkt.is_resent: # protocol messages and resent packets have higher priority self._send_prio_buffer.append(pkt) else: self._send_buffer.append(pkt) # sending occurs on the next write event if not self._write_active: self._write_active = True self.uc_reactor.add_write() return def _packet_sent(self, pkt): """Helper method called after the given packet has been sent. Shrinks backlog if necessary then adds packet to backlog and (re-)sets the heartbeat timer. """ # shrink backlog l_diff = len(self._backlog) - self.MAX_BACKLOG if l_diff > 0: self._backlog = self._backlog[l_diff:] if not pkt.is_resent: # mark passed packet as last sent packet for heartbeat # call event listners and reset heartbeat time self._last_sent = pkt self._backlog.append(pkt) self._notify_handlers('packet_sent', pkt) self.reset_heartbeat() # check for seq overflow if self.seq > 2000000000: # emit sequence_overflow self._notify_handlers('sequence_overflow', None) return def _send_packets(self, packets): uc_send = self.uc_reactor.send notify = self._notify_handlers p_sent = self._packet_sent for p in packets: uc_send(str(p), p.dest_addr) notify('__sent_packet__', p) if not (p.is_protocol or p.is_resent): # new normal packet, post-process it p_sent(p) return def on_write(self): """Event handler called when reactor can send more data. Will send at most MAX_BURST packets at once. """ # first send all priority packets collected out_list = self._send_prio_buffer[:self.MAX_BURST] self._send_prio_buffer = self._send_prio_buffer[self.MAX_BURST:] out_len = len(out_list) if out_list: self._send_packets(out_list) # try to send normal packets if out_len < self.MAX_BURST: r = self.MAX_BURST - out_len out_list = self._send_buffer[:r] self._send_buffer = self._send_buffer[r:] self._send_packets(out_list) if self._send_prio_buffer or self._send_buffer: # more data to send, re-activate write events self._write_active = True self.uc_reactor.add_write() else: # nothing to do, disable write events self._write_active = False self.uc_reactor.del_write() return def _get_oldest_sender(self): """Helper returning the sender with the smallest last_heartbeat value or None. """ if not self._peers: return None lp = operator.attrgetter('last_packet') senders = sorted(self._peers.values(), key=lp) return senders[0] def _hb_generator(self): """Helper method that creates a generator for the heartbeat delay interval. Uses yield expression! """ try: self._hb_times except AttributeError: self._hb_times = self._def_times self._hb_index = 0 while True: times = self._hb_times try: t_next = times[self._hb_index] except IndexError: t_next = times[-1] self._hb_index = len(times) - 1 ## t_next = rnd_delay_time(t_next) ret = (yield t_next) if ret: self._hb_index = 0 elif self._hb_index >= 0: self._hb_index += 1 return def _next_hb_time(self, reset=None): """Helper method returning the delay time for the next heartbeat. If reset is set, the heartbeat delay is reset to the smallest value. Uses .send method for generators! """ ret = self._hb_gen.send(reset) return ret def _check_sender_timeouts(self): """Helper method to check for senders who timed out. Notify 'missing_heartbeat' listners of senders who's last package was received more then their current heartbeat delay in the past. """ now = time.time() pv = self._peers.values() timed_out = [s for s in pv if s.heartbeat_overdue(now)] # notify listners of missing senders # the action handler is responsible for removing the sender, if wanted for sender in timed_out: self._notify_handlers('missing_heartbeat', sender) return def on_heartbeat(self): """Event handler called from heartbeat timer. Send next heartbeat if needed and adjust timer interval. """ self._check_sender_timeouts() if self.seq < 0 or self._send_buffer: # nothing sent so far t_next = 0.01 elif not self._last_sent: t_next = 0.01 elif self.seq == (self._last_sent.seq + 1): # no change, use next HB interval # send HB packet p_last = self._last_sent seq = p_last.seq # adjust timer t_next = self._next_hb_time() p = HeartbeatPacket(seq, heartbeat=self._hb_index) self.send(p) self._notify_handlers('sent_heartbeat', p) else: # something changed, reset heartbeat t_next = self._next_hb_time(True) return t_next def _hb_timer_tick(self): self._hb_timer.delete() t_next = self.on_heartbeat() self._hb_timer = event.timeout(t_next, self._hb_timer_tick) return def _chk_timer_tick(self): self._chk_timer.delete() self._check_sender_timeouts() self._chk_timer = event.timeout(0.01, self._chk_timer_tick) return def reset_heartbeat(self): """Reset heartbeat time to lowest value. """ if not self._hb_timer: return self._hb_timer.delete() t_next = self._next_hb_time(True) self._hb_timer = event.timeout(t_next, self._hb_timer_tick) return t_next def _deliver_look_ahead(self, sender, start=None): """Helper to deliver packets from this look-ahead cache that are in sequence starting at start. If start is None, sender.seq+1 is assumed. Returns last delivered seq or None if nothing was sent. """ in_seq = sender.get_look_ahead(start) if not in_seq: # nothing found return None last_seen = in_seq[-1].seq # deliver packets for p in in_seq: self._notify_handlers('new_packet', p) # purge cache sender.purge_look_ahead(last_seen) return last_seen def _send_nack_range(self, sender, count=1, now=None): """Helper to send out NACK packets. """ if now is None: now = time.time() sent_nacks = sender.sent_nacks n_seq = sender.seq + 1 while count > 0: if n_seq in sent_nacks: # check for nack resend t_nack, t_first = sent_nacks[n_seq] if (now - t_first) >= sender.nack_timeout: # old NACK, drop it del sent_nacks[n_seq] elif (now - t_nack) >= sender.nack_resend_time: # time to re-send NACK p = NackPacket(n_seq) self.send(p, sender.address) self._notify_handlers('sent_nack', p) sent_nacks[n_seq] = (now, t_first) elif len(sent_nacks) < sender.max_nacks: # new NACK to send p = NackPacket(n_seq) self.send(p, sender.address) self._notify_handlers('sent_nack', p) sent_nacks[n_seq] = (now, now) # next in sequence n_seq += 1 count -= 1 return def _deal_with_nack(self, pkt): """Helper method to deal with received NACK packet. If the requested packet is in the backlog, it's sent to the address the nack has been received from. If not a LOST packet is sent. """ sender = pkt.sender # mark sender as still alive sender.touch() # notify interested parties self._notify_handlers('got_nack', pkt) # check for LOST condition addr = pkt.src_addr sent = False if not self._backlog: # no backlog # send RESET packet p = packet.Packet(self.seq) p.flags |= packet.RESET self.send(p, addr) self._notify_handlers('sent_lost', p) return p0 = self._backlog[0] if pkt.seq < p0.seq: # if the requested seq is smaller than the smallest seq in # our backlog send a LOST packet with the lowest available # seq to cut down further nacks seq = p0.seq p = LostPacket(seq) # targeted to addr, others wont see it! self.send(p, addr) self._notify_handlers('sent_lost', p) return # packet still in backlog # try to send it for p in self._backlog: if pkt.seq == p.seq: # re-send requested packets p.flags |= packet.RESENT # unicast send! self.send(p, addr) sent = True break if not sent: # snafu # packet not in backlog anymore # send LostPacket p0 = self._backlog[0] p = LostPacket(p0.seq) self.send(p, addr) self._notify_handlers('sent_lost', p) else: # speed up recovery self.reset_heartbeat() return def _deal_with_lost(self, pkt): """Helper method to deal with a received LOST packet. Clean up expected packet sequences (aka sent nacks) and shrink the look-ahead cache. Then try to deliver packets from the look-ahead cache. """ sender = pkt.sender # packets with a seq lower then the given can't be recovered # remove them from outstanding nack list and adjust last_seen sender.purge_sent_nacks(pkt.seq) sender.purge_look_ahead(pkt.seq) # notify listners before delivery of data packets sender.touch(pkt.seq - 1) self._notify_handlers('got_lost', pkt) # send nack for smallest available packet self._send_nack_range(sender) return def _deal_with_reset(self, pkt): """Helper method to deal with a received RESET packet. Will reset (aka clear) all data for this sender and set senders seq to the one in the packet. """ sender = pkt.sender # forget all information on sender sender.reset_all(pkt.seq) self._notify_handlers('got_reset', pkt) return def _deal_with_heartbeat(self, pkt): """Helper method to deal with a received HEARTBEAT packet. """ sender = pkt.sender self._notify_handlers('got_heartbeat', pkt) now = time.time() last_seen = sender.seq if last_seen < 0 and self.no_sync: # avoid sending nack on first heartbeat of sender when no_sync is set sender.touch(pkt.seq, heartbeat=pkt.heartbeat) return sender.touch(heartbeat=pkt.heartbeat, now=now) pdiff = pkt.seq - last_seen if pdiff > 0: # we missed at least one packet # send NACKs self._send_nack_range(sender, pdiff, now) return def _deal_with_hb_times(self, pkt): """Helper method to deal with a received HB_TIMES packet. """ sender = pkt.sender now = time.time() times = packet.decode_times(pkt) sender.set_heartbeat_times(times) sender.touch(heartbeat=pkt.heartbeat, now=now) return def _protocol_packet(self, pkt): """Handle protocol specific packet received from addr. """ if pkt.flags & packet.NACK: self._deal_with_nack(pkt) elif pkt.flags & packet.LOST: self._deal_with_lost(pkt) elif pkt.flags & packet.HEARTBEAT: self._deal_with_heartbeat(pkt) elif pkt.flags & packet.HB_TIMES: self._deal_with_hb_times(pkt) elif pkt.flags & packet.RESET: self._deal_with_reset(pkt) elif pkt.flags & packet.ACK: # shouldn't we be happy? # sure, but we ignore ACK packets pass return def _data_packet(self, pkt): """Deal with ordinary data packet. Checks for and deals with missing packets. """ sender = pkt.sender if sender.seq < 0 and self.no_sync: # don't send nacks on first packet sender.seq = pkt.seq - 1 now = time.time() pdiff = pkt.seq - sender.seq sent_nacks = sender.sent_nacks look_ahead = sender.look_ahead last_seen = sender.seq if pdiff == 1: # received next packet in sequence # deal with it if pkt.seq in sent_nacks: del sent_nacks[pkt.seq] self._notify_handlers('new_packet', pkt) sender.seq = pkt.seq # try to deliver cached packets last_seen = self._deliver_look_ahead(sender, pkt.seq + 1) sender.touch(seq=last_seen, heartbeat=pkt.heartbeat) elif pdiff > 1: # we missed at least one packet # forward caching of packets # reduces the number of nacks needed look_ahead[pkt.seq] = pkt # sent NACKs for missing packets self._send_nack_range(sender, pdiff, now) sender.touch(heartbeat=pkt.heartbeat) else: # ouch... # but we mark sender as still alive sender.touch(heartbeat=pkt.heartbeat) return def on_packet(self, p): """Event handler called when reactor received a new rmc packet. p is the packet """ if p.src_addr == self.local_addr: # we get our own packets due to the multicast mechanism # ignore them return if p.recv_addr == self.local_addr: # unicast aka private p.flags |= packet.PRIVATE # fetch/create Peer instance for packet sender sender = self._peers.get(p.src_addr, None) if not sender: # new (unknown) sender notification sender = Peer(p.src_addr, times=self._def_times) self._peers[p.src_addr] = sender # new sender, notify listners self._notify_handlers('new_sender', sender) p.sender = sender # call event handler for *all* packets, debugging only # otherwise it'll be slow self._notify_handlers('__received_packet__', p) # dispatch packet if p.flags & packet.PROTOFLAGS: self._protocol_packet(p) else: self._data_packet(p) return
def __init__(self, uc_socket, mc_socket=None): """Setup rmc handler. uc_socket: UdpSocket bound to local unicast address mc_socket: UdpSocket bound to multicast address, optional Attributes: mc_socket: UdpSocket bound to multicast address uc_socket: UdpSocket bound to local unicast address mc_reactor: RmcReactor for multicast address (receive only) uc_reactor: RmcReactor for unicast address (receive/send) dest_addr: ip/port of multicast channel local_addr: ip/port for unicast seq: current sequence no. for outgoing packets no_sync: if True don't send NACKs on first heartbeat for a sender silent : if True don't send data, never ever, just listen """ # create multicast socket ## self.mc_socket = UdpSocket((mc_addr[0], mc_addr[1])) ## self.mc_socket.multicast_interface = mc_interface ## self.mc_socket.join_mcast_group(mc_addr[0]) self.mc_socket = mc_socket self.uc_socket = uc_socket # create reactors for both sockets # and attach them to this protocol handler self.uc_reactor = RmcReactor(self.uc_socket, self) self.local_addr = self.uc_socket.getsockname() # if multicast socket is None # sending will use the unicast socket as fallback if mc_socket is None: self.mc_reactor = self.uc_reactor ## self.mc_socket.multicast_interface = mc_interface else: self.mc_reactor = RmcReactor(mc_socket, self) self.mc_socket.multicast_loop = True # get/assign addresses self.dest_addr = self.mc_socket.getsockname() # send_buffer is a list of packets to send out # send_prio_buffer is for outgoing packets w/ high priority self._send_buffer = [] self._send_prio_buffer = [] # sequence counter for outgoing data packets self.seq = 0 # remember last data packet sent self._last_sent = None # keep track of peers self._peers = {} # our outgoing backlog self._backlog = [] # activate reactors self.mc_reactor.add_read() self.uc_reactor.add_read() # initialize generator for hearbeat times self._hb_gen = self._hb_generator() self._hb_gen.next() # mapping to keep track of event subscriptions self._ev_handler = {} # flag if write event is active self._write_active = False # flag when set, no sync on first heartbeat of a sender is done self.no_sync = False # flag when set, no data will be sent (ever!) self._silent = False # prepare continous timer ticks self._hb_timer = event.timeout(0.1, self._hb_timer_tick) self._chk_timer = event.timeout(0.01, self._chk_timer_tick) # flag showing if this handler does receive data via multicast self.is_active = True return