class EagerReliableBroadcast(Broadcast): """This class implements a eager reliable broadcast. Uses: - BestEffortBroadcast (in theory) - PerfectLink (in practice) This class does not use the BestEffortBroadcast class in its implementation for convenience. It acts as if, though. """ def __init__(self, link, max_concurrent_messages=20): super().__init__(link) self.delivered = [None] * max_concurrent_messages self.delivered_cycle = 0 self.timestamp = 0 self.logger = Logging(self.process_number, "ERB") def register_delivered(self, message): self.delivered[self.delivered_cycle] = message self.delivered_cycle = (self.delivered_cycle + 1) % len(self.delivered) def broadcast(self, callback_id, args=(), kwargs={}): self.logger.log_debug(f"Broadcasting {(kwargs)}") message = (self.timestamp, self.process_number, callback_id, args, kwargs) self.timestamp += 1 self._broadcast(message) def receive(self, source_number, timestamp, original_source, callback_id, args=(), kwargs={}): message = (timestamp, original_source, callback_id, args, kwargs) if message not in self.delivered: self.logger.log_debug( f"Receiving {(args, kwargs)} from {original_source}") self.register_delivered(message) self.callback(callback_id, args=args, kwargs=kwargs) self._broadcast(message) def _broadcast(self, message): for peer in self.peers: self.send(peer, self.receive, args=message)
class BestEffortBroadcast(Broadcast): """This class implements a best effort broadcast abstraction. Uses: - PerfectLink """ def __init__(self, link): super().__init__(link) self.logger = Logging(self.process_number, "BEB") def broadcast(self, callback_id, args=(), kwargs={}): self.logger.log_debug(f"Broadcasting {(args, kwargs)}") for peer in self.peers: self.send(peer, self.receive, args=(callback_id, args, kwargs)) def receive(self, source_number, callback_id, args=(), kwargs={}): self.logger.log_debug( f"Receiving {(args, kwargs)} from {source_number}") self.callback(callback_id, args=args, kwargs=kwargs)
class LeaderElection(Subscriptable): """ This class implement a hierarchical leader election abstraction. Uses: - PerfectFailureDetector - HierarchicalConsensus A peer's rank is greater than another's iff its process number is strictly lower than the other's. All sbscribed methods are called whenever a new leader is elected. """ def __init__(self, pfd, hco): super().__init__() self.process_number = hco.process_number self.pfd = pfd self.pfd.subscribe_abstraction(self, self.peer_failure) self.hco = hco self.hco.subscribe_abstraction(self, self.decided) self.peers = {self.process_number} self.detected = set() self.leader = None self.in_election = False self.logger = Logging(self.process_number, "LEL") def start(self): super().start() self.election() def add_peers(self, *peers): self.peers.update(peers) self.pfd.add_peers(*peers) self.hco.add_peers(*peers) def peer_failure(self, process_number): self.logger.log_debug(f"Peer {process_number} crashed") self.detected.add(process_number) self.election() def election(self): if not self.in_election: self.logger.log_debug(f"New election") self.in_election = True self.leader = None leader = min(self.peers - self.detected) self.hco.trigger_event(self.hco.propose, kwargs={"value": leader}) def decided(self, value): self.in_election = False if value in (self.peers - self.detected): self.logger.log_debug(f"New leader {value}") self.leader = value self.call_callbacks(self.leader) else: self.election()
class MajorityVoting(Abstraction): """This class implements a majority voting abstraction. Uses: - PerfectFailureDetector - EagerReliableBroadcast - HierchicalConsensus Since this is the top-level class, it instantiates all the classes it requires (and their requirements). It is also not meant to be used by an abstraction (but still can), but by, say, a flight computer object. This class is callable, and its entrypoint is its __call__ method. """ def __init__(self, process_number, decide_callback, deliver_callback): super().__init__() self.process_number = process_number self.decide_callback = decide_callback self.deliver_callback = deliver_callback self.link = PerfectLink(self.process_number) self.pfd = PerfectFailureDetector(self.link) self.pfd.subscribe_abstraction(self, self.peer_failure) self.erb = EagerReliableBroadcast(self.link) self.broadcast = self.erb.register_abstraction(self) self.beb = BestEffortBroadcast(self.link) self.hco = HierarchicalConsensus(self.link, self.pfd, self.beb) self.hco.subscribe_abstraction(self, self.consensus_decided) self.lel_hco = HierarchicalConsensus(self.link, self.pfd, self.beb) self.lel = LeaderElection(self.pfd, self.lel_hco) self.lel.subscribe_abstraction(self, self.new_leader) self.leader = None self.peers = {self.process_number} self.detected = set() self.erb.add_peers(self.process_number) self.votes = {} self.voted = {peer: False for peer in self.peers} self.finished_election = Event() self.finished_consensus = Event() self.finished_consensus.set() self.consensus_result = None self.proposition = None self.logger = Logging(self.process_number, "VOT") def add_peers(self, *peers): self.peers.update(peers) self.pfd.add_peers(*peers) self.beb.add_peers(*peers) self.hco.add_peers(*peers) self.lel.add_peers(*peers) self.erb.add_peers(*peers) self.voted.update({peer: False for peer in self.peers}) def start(self): super().start() self.link.start() self.pfd.start() self.erb.start() self.beb.start() self.hco.start() self.lel_hco.start() self.lel.start() def stop(self): super().stop() self.link.stop() self.pfd.stop() self.erb.stop() self.beb.stop() self.hco.stop() self.lel_hco.stop() self.lel.stop() def peer_failure(self, process_number): self.logger.log_debug(f"Peer {process_number} crashed") if process_number == self.leader: self.leader = None self.finished_election.clear() self.detected.add(process_number) self.erb.peers.remove(process_number) self.finished_vote(process_number) def new_leader(self, process_number): self.logger.log_debug(f"New leader {process_number}") self.leader = process_number self.finished_election.set() def new_vote(self, source_number, value): if source_number != self.leader: return self.logger.log_debug( f"Received new vote request {value} from {source_number}") self.finished_consensus.clear() self.proposition = value vote = self.decide_callback(value) self.broadcast(self.vote_receive, kwargs={"vote": vote}) def vote_receive(self, source_number, vote): self.logger.log_debug(f"Received vote {vote} from {source_number}") if vote in self.votes: self.votes[vote] += 1 else: self.votes[vote] = 1 self.finished_vote(source_number) def finished_vote(self, process_number): self.voted[process_number] = True if all(self.voted.values()): self.logger.log_debug(f"Voting finished: {self.votes}") max_vote = max(self.votes, key=self.votes.get) self.votes.clear() self.voted = {peer: False for peer in self.peers - self.detected} self.hco.trigger_event(self.hco.propose, kwargs={"value": max_vote}) def consensus_decided(self, value): self.logger.log_debug(f"Consensus decided on {value}") self.consensus_result = value if self.consensus_result: self.deliver_callback(self.proposition) self.finished_consensus.set() # Entrypoints def vote(self, value): # Waiting for election if not self.finished_election.wait(self.TIMEOUT / 3): return False if self.leader != self.process_number or not self.alive: return False self.logger.log_debug(f"New vote on: {value}") # Waiting last consensus if not self.finished_consensus.wait(self.TIMEOUT / 3): return False self.finished_consensus.clear() self.broadcast(self.new_vote, kwargs={"value": value}) if not self.finished_consensus.wait(self.TIMEOUT / 3): return False return self.consensus_result def get_leader(self): if self.finished_election.wait(self.TIMEOUT / 3): return self.leader
class PerfectLink(Registrable): """This class implements a perfect link. This class uses unix domain sockets for IPC operations. It is a Registrable such that it may serve multiple abstractions. An Abstraction object must go through the generate_abstraction_caller method instead of using the send method directly. """ SEND = 0 MAX_LEN = 1024 def __init__(self, process_number): super().__init__() self.process_number = process_number self.create_socket() self.listener = Thread(target=self.receive) self.logger = Logging(self.process_number, "LINK") def start(self): super().start() self.listener.start() def create_socket(self): server_address = self.get_address(self.process_number) try: os.unlink(server_address) except OSError: if os.path.exists(server_address): raise self.socket = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) self.socket.bind(server_address) self.socket.settimeout(self.TIMEOUT) def send(self, destination_process, callback_id, args=(), kwargs={}): message = (callback_id, args, kwargs) if self.alive: data = pickle.dumps(message) if len(data) > self.MAX_LEN: raise Exception( f"Message exceding maximum length of {self.MAX_LEN} bytes, received {len(data)} bytes" ) self.logger.log_debug( f"Sending {(args, kwargs)} to {destination_process}") try: self.socket.sendto(data, self.get_address(destination_process)) except Exception as e: self.logger.log_debug( f"Message {message} for {destination_process} dropped") else: self.logger.log_debug( f"Not send {message} to {destination_process}") def receive(self): while self.alive: try: data, source = self.socket.recvfrom(self.MAX_LEN) except socket.timeout: continue else: callback_id, args, kwargs = pickle.loads(data) source_number = self.get_process(source) self.logger.log_debug( f"Received {(args, kwargs)} from {source_number}") self.callback(callback_id, args=args, kwargs=kwargs) self.socket.close() self.logger.log_debug(f"is done") def generate_abstraction_caller(self, callback_id): def sender(destination_process, event, args=(), kwargs={}): event_name = self.stringify_event(event) args = (event_name, self.process_number, *args) self.trigger_event(self.send, args=(destination_process, callback_id, args, kwargs)) return sender def get_address(self, process_number): return f"/tmp/fairlosslink{process_number}.socket" def get_process(self, address): return int(re.findall("[0-9]+", address)[0])
class HierarchicalConsensus(Consensus): """This class implements the hierchical consensus algorithm. Uses: - PerfectFailureDetector - BestEffortBroadcast A peer's rank is greater than another's iff its process number is strictly lower than the other's. The propose method is the entry point to the consensus algorithm. When the consensus is finished, the subscribed callbacks are called. """ def __init__(self, link, pfd, beb): super().__init__(link) self.beb = beb self.broadcast = self.beb.register_abstraction(self) self.pfd = pfd self.pfd.subscribe_abstraction(self, self.peer_failure) self.peers = {self.process_number} self.beb.add_peers(self.process_number) self.detected = set() self.reset() self.finished_peers = {peer: False for peer in self.peers} self.logger = Logging(self.process_number, "HCO") def reset(self): self.round = 0 self.proposal = None self.proposer = -1 self.delivered = {peer: False for peer in self.peers} self.broadcasting = False def add_peers(self, *peers): self.beb.add_peers(*peers) self.pfd.add_peers(*peers) self.peers.update(peers) self.delivered.update({peer: False for peer in peers}) self.finished_peers.update({peer: False for peer in peers}) def peer_failure(self, process_number): self.logger.log_debug(f"Peer {process_number} crashed") self.detected.add(process_number) self.round_update() self.finished(process_number) def propose(self, value): self.logger.log_debug(f"New proposal {value}") if self.proposal is None: self.proposal = value self.round_update() def round_update(self): while self.round < len(self.peers) and (self.round in self.detected or self.delivered[self.round]): self.round += 1 if self.round == len(self.peers): self.reset() self.broadcast(self.finished) elif (self.round == self.process_number and self.proposal is not None and not self.broadcasting): self.broadcasting = True self.decided = self.proposal self.broadcast(self.receive, args=(self.decided, )) def receive(self, source_number, value): if source_number in self.detected: return self.logger.log_debug( f"Process {source_number} has decided on {value}") if source_number < self.process_number and source_number > self.proposer: self.proposal = value self.proposer = source_number self.delivered[source_number] = True self.round_update() def finished(self, source_number): self.finished_peers[source_number] = True if all(self.finished_peers.values()): self.logger.log_debug(f"Consensus finished") self.call_callbacks(self.decided) self.finished_peers = { peer: False for peer in self.peers - self.detected }
class PerfectFailureDetector(Subscriptable): """This class implements the perfect failure detector abstraction. Uses: - PĂ«rfectLink This class uses hearbeats that require a response. If no response frome a peer came back in time, the peer is assumed to have failed. When a peer fails, the subscribed callbacks are called. Timeout is chosen to be the Abstraction class' TIMEOUT attribute divided by 10. Perhaps taking twice the round-trip delay time would have been a better idea? """ def __init__(self, link): super().__init__() self.link = link self.process_number = self.link.process_number self.send = self.link.register_abstraction(self) self.peers = set() self.detected = set() self.correct = set() self.lock = Lock() self.worker = Thread(target=self.detect_failures) self.logger = Logging(self.process_number, "PFD") def start(self): super().start() self.worker.start() def add_peers(self, *peers): self.peers.update(peers) def request(self, source_number): self.logger.log_debug(f"Request from {source_number}") self.send(source_number, self.reply) def reply(self, source_number): self.logger.log_debug(f"Reply from {source_number}") with self.lock: self.correct.add(source_number) def detect_failures(self): while self.alive: self.send_heartbeats() time.sleep(self.TIMEOUT / 10) self.timeout() def send_heartbeats(self): for peer in self.peers - self.detected: self.send(peer, self.request) def timeout(self): with self.lock: for peer in self.peers - self.correct - self.detected: self.logger.log_debug(f"Peer {peer} crashed") self.detected.add(peer) self.call_callbacks(peer) self.correct.clear()