class EagerReliableBroadcast(Broadcast):
"""This class implements a eager reliable broadcast.
Uses:
- BestEffortBroadcast (in theory)
- PerfectLink (in practice)
This class does not use the BestEffortBroadcast class in its implementation
for convenience. It acts as if, though.
"""
def __init__(self, link, max_concurrent_messages=20):
super().__init__(link)
self.delivered = [None] * max_concurrent_messages
self.delivered_cycle = 0
self.timestamp = 0
self.logger = Logging(self.process_number, "ERB")
def register_delivered(self, message):
self.delivered[self.delivered_cycle] = message
self.delivered_cycle = (self.delivered_cycle + 1) % len(self.delivered)
def broadcast(self, callback_id, args=(), kwargs={}):
self.logger.log_debug(f"Broadcasting {(kwargs)}")
message = (self.timestamp, self.process_number, callback_id, args,
kwargs)
self.timestamp += 1
self._broadcast(message)
def receive(self,
source_number,
timestamp,
original_source,
callback_id,
args=(),
kwargs={}):
message = (timestamp, original_source, callback_id, args, kwargs)
if message not in self.delivered:
self.logger.log_debug(
f"Receiving {(args, kwargs)} from {original_source}")
self.register_delivered(message)
self.callback(callback_id, args=args, kwargs=kwargs)
self._broadcast(message)
def _broadcast(self, message):
for peer in self.peers:
self.send(peer, self.receive, args=message)
class BestEffortBroadcast(Broadcast):
"""This class implements a best effort broadcast abstraction.
Uses:
- PerfectLink
"""
def __init__(self, link):
super().__init__(link)
self.logger = Logging(self.process_number, "BEB")
def broadcast(self, callback_id, args=(), kwargs={}):
self.logger.log_debug(f"Broadcasting {(args, kwargs)}")
for peer in self.peers:
self.send(peer, self.receive, args=(callback_id, args, kwargs))
def receive(self, source_number, callback_id, args=(), kwargs={}):
self.logger.log_debug(
f"Receiving {(args, kwargs)} from {source_number}")
self.callback(callback_id, args=args, kwargs=kwargs)
class LeaderElection(Subscriptable):
""" This class implement a hierarchical leader election abstraction.
Uses:
- PerfectFailureDetector
- HierarchicalConsensus
A peer's rank is greater than another's iff its process number is strictly
lower than the other's. All sbscribed methods are called whenever a new
leader is elected.
"""
def __init__(self, pfd, hco):
super().__init__()
self.process_number = hco.process_number
self.pfd = pfd
self.pfd.subscribe_abstraction(self, self.peer_failure)
self.hco = hco
self.hco.subscribe_abstraction(self, self.decided)
self.peers = {self.process_number}
self.detected = set()
self.leader = None
self.in_election = False
self.logger = Logging(self.process_number, "LEL")
def start(self):
super().start()
self.election()
def add_peers(self, *peers):
self.peers.update(peers)
self.pfd.add_peers(*peers)
self.hco.add_peers(*peers)
def peer_failure(self, process_number):
self.logger.log_debug(f"Peer {process_number} crashed")
self.detected.add(process_number)
self.election()
def election(self):
if not self.in_election:
self.logger.log_debug(f"New election")
self.in_election = True
self.leader = None
leader = min(self.peers - self.detected)
self.hco.trigger_event(self.hco.propose, kwargs={"value": leader})
def decided(self, value):
self.in_election = False
if value in (self.peers - self.detected):
self.logger.log_debug(f"New leader {value}")
self.leader = value
self.call_callbacks(self.leader)
else:
self.election()
class MajorityVoting(Abstraction):
"""This class implements a majority voting abstraction.
Uses:
- PerfectFailureDetector
- EagerReliableBroadcast
- HierchicalConsensus
Since this is the top-level class, it instantiates all the classes it
requires (and their requirements). It is also not meant to be used by an
abstraction (but still can), but by, say, a flight computer object. This
class is callable, and its entrypoint is its __call__ method.
"""
def __init__(self, process_number, decide_callback, deliver_callback):
super().__init__()
self.process_number = process_number
self.decide_callback = decide_callback
self.deliver_callback = deliver_callback
self.link = PerfectLink(self.process_number)
self.pfd = PerfectFailureDetector(self.link)
self.pfd.subscribe_abstraction(self, self.peer_failure)
self.erb = EagerReliableBroadcast(self.link)
self.broadcast = self.erb.register_abstraction(self)
self.beb = BestEffortBroadcast(self.link)
self.hco = HierarchicalConsensus(self.link, self.pfd, self.beb)
self.hco.subscribe_abstraction(self, self.consensus_decided)
self.lel_hco = HierarchicalConsensus(self.link, self.pfd, self.beb)
self.lel = LeaderElection(self.pfd, self.lel_hco)
self.lel.subscribe_abstraction(self, self.new_leader)
self.leader = None
self.peers = {self.process_number}
self.detected = set()
self.erb.add_peers(self.process_number)
self.votes = {}
self.voted = {peer: False for peer in self.peers}
self.finished_election = Event()
self.finished_consensus = Event()
self.finished_consensus.set()
self.consensus_result = None
self.proposition = None
self.logger = Logging(self.process_number, "VOT")
def add_peers(self, *peers):
self.peers.update(peers)
self.pfd.add_peers(*peers)
self.beb.add_peers(*peers)
self.hco.add_peers(*peers)
self.lel.add_peers(*peers)
self.erb.add_peers(*peers)
self.voted.update({peer: False for peer in self.peers})
def start(self):
super().start()
self.link.start()
self.pfd.start()
self.erb.start()
self.beb.start()
self.hco.start()
self.lel_hco.start()
self.lel.start()
def stop(self):
super().stop()
self.link.stop()
self.pfd.stop()
self.erb.stop()
self.beb.stop()
self.hco.stop()
self.lel_hco.stop()
self.lel.stop()
def peer_failure(self, process_number):
self.logger.log_debug(f"Peer {process_number} crashed")
if process_number == self.leader:
self.leader = None
self.finished_election.clear()
self.detected.add(process_number)
self.erb.peers.remove(process_number)
self.finished_vote(process_number)
def new_leader(self, process_number):
self.logger.log_debug(f"New leader {process_number}")
self.leader = process_number
self.finished_election.set()
def new_vote(self, source_number, value):
if source_number != self.leader:
return
self.logger.log_debug(
f"Received new vote request {value} from {source_number}")
self.finished_consensus.clear()
self.proposition = value
vote = self.decide_callback(value)
self.broadcast(self.vote_receive, kwargs={"vote": vote})
def vote_receive(self, source_number, vote):
self.logger.log_debug(f"Received vote {vote} from {source_number}")
if vote in self.votes:
self.votes[vote] += 1
else:
self.votes[vote] = 1
self.finished_vote(source_number)
def finished_vote(self, process_number):
self.voted[process_number] = True
if all(self.voted.values()):
self.logger.log_debug(f"Voting finished: {self.votes}")
max_vote = max(self.votes, key=self.votes.get)
self.votes.clear()
self.voted = {peer: False for peer in self.peers - self.detected}
self.hco.trigger_event(self.hco.propose,
kwargs={"value": max_vote})
def consensus_decided(self, value):
self.logger.log_debug(f"Consensus decided on {value}")
self.consensus_result = value
if self.consensus_result:
self.deliver_callback(self.proposition)
self.finished_consensus.set()
# Entrypoints
def vote(self, value):
# Waiting for election
if not self.finished_election.wait(self.TIMEOUT / 3):
return False
if self.leader != self.process_number or not self.alive:
return False
self.logger.log_debug(f"New vote on: {value}")
# Waiting last consensus
if not self.finished_consensus.wait(self.TIMEOUT / 3):
return False
self.finished_consensus.clear()
self.broadcast(self.new_vote, kwargs={"value": value})
if not self.finished_consensus.wait(self.TIMEOUT / 3):
return False
return self.consensus_result
def get_leader(self):
if self.finished_election.wait(self.TIMEOUT / 3):
return self.leader
class PerfectLink(Registrable):
"""This class implements a perfect link.
This class uses unix domain sockets for IPC operations. It is a Registrable
such that it may serve multiple abstractions. An Abstraction object must go
through the generate_abstraction_caller method instead of using the send
method directly.
"""
SEND = 0
MAX_LEN = 1024
def __init__(self, process_number):
super().__init__()
self.process_number = process_number
self.create_socket()
self.listener = Thread(target=self.receive)
self.logger = Logging(self.process_number, "LINK")
def start(self):
super().start()
self.listener.start()
def create_socket(self):
server_address = self.get_address(self.process_number)
try:
os.unlink(server_address)
except OSError:
if os.path.exists(server_address):
raise
self.socket = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM)
self.socket.bind(server_address)
self.socket.settimeout(self.TIMEOUT)
def send(self, destination_process, callback_id, args=(), kwargs={}):
message = (callback_id, args, kwargs)
if self.alive:
data = pickle.dumps(message)
if len(data) > self.MAX_LEN:
raise Exception(
f"Message exceding maximum length of {self.MAX_LEN} bytes, received {len(data)} bytes"
)
self.logger.log_debug(
f"Sending {(args, kwargs)} to {destination_process}")
try:
self.socket.sendto(data, self.get_address(destination_process))
except Exception as e:
self.logger.log_debug(
f"Message {message} for {destination_process} dropped")
else:
self.logger.log_debug(
f"Not send {message} to {destination_process}")
def receive(self):
while self.alive:
try:
data, source = self.socket.recvfrom(self.MAX_LEN)
except socket.timeout:
continue
else:
callback_id, args, kwargs = pickle.loads(data)
source_number = self.get_process(source)
self.logger.log_debug(
f"Received {(args, kwargs)} from {source_number}")
self.callback(callback_id, args=args, kwargs=kwargs)
self.socket.close()
self.logger.log_debug(f"is done")
def generate_abstraction_caller(self, callback_id):
def sender(destination_process, event, args=(), kwargs={}):
event_name = self.stringify_event(event)
args = (event_name, self.process_number, *args)
self.trigger_event(self.send,
args=(destination_process, callback_id, args,
kwargs))
return sender
def get_address(self, process_number):
return f"/tmp/fairlosslink{process_number}.socket"
def get_process(self, address):
return int(re.findall("[0-9]+", address)[0])
class HierarchicalConsensus(Consensus):
"""This class implements the hierchical consensus algorithm.
Uses:
- PerfectFailureDetector
- BestEffortBroadcast
A peer's rank is greater than another's iff its process number is strictly
lower than the other's. The propose method is the entry point to the
consensus algorithm. When the consensus is finished, the subscribed
callbacks are called.
"""
def __init__(self, link, pfd, beb):
super().__init__(link)
self.beb = beb
self.broadcast = self.beb.register_abstraction(self)
self.pfd = pfd
self.pfd.subscribe_abstraction(self, self.peer_failure)
self.peers = {self.process_number}
self.beb.add_peers(self.process_number)
self.detected = set()
self.reset()
self.finished_peers = {peer: False for peer in self.peers}
self.logger = Logging(self.process_number, "HCO")
def reset(self):
self.round = 0
self.proposal = None
self.proposer = -1
self.delivered = {peer: False for peer in self.peers}
self.broadcasting = False
def add_peers(self, *peers):
self.beb.add_peers(*peers)
self.pfd.add_peers(*peers)
self.peers.update(peers)
self.delivered.update({peer: False for peer in peers})
self.finished_peers.update({peer: False for peer in peers})
def peer_failure(self, process_number):
self.logger.log_debug(f"Peer {process_number} crashed")
self.detected.add(process_number)
self.round_update()
self.finished(process_number)
def propose(self, value):
self.logger.log_debug(f"New proposal {value}")
if self.proposal is None:
self.proposal = value
self.round_update()
def round_update(self):
while self.round < len(self.peers) and (self.round in self.detected
or self.delivered[self.round]):
self.round += 1
if self.round == len(self.peers):
self.reset()
self.broadcast(self.finished)
elif (self.round == self.process_number and self.proposal is not None
and not self.broadcasting):
self.broadcasting = True
self.decided = self.proposal
self.broadcast(self.receive, args=(self.decided, ))
def receive(self, source_number, value):
if source_number in self.detected:
return
self.logger.log_debug(
f"Process {source_number} has decided on {value}")
if source_number < self.process_number and source_number > self.proposer:
self.proposal = value
self.proposer = source_number
self.delivered[source_number] = True
self.round_update()
def finished(self, source_number):
self.finished_peers[source_number] = True
if all(self.finished_peers.values()):
self.logger.log_debug(f"Consensus finished")
self.call_callbacks(self.decided)
self.finished_peers = {
peer: False
for peer in self.peers - self.detected
}
class PerfectFailureDetector(Subscriptable):
"""This class implements the perfect failure detector abstraction.
Uses:
- PërfectLink
This class uses hearbeats that require a response. If no response frome a
peer came back in time, the peer is assumed to have failed. When a peer
fails, the subscribed callbacks are called. Timeout is chosen to be the
Abstraction class' TIMEOUT attribute divided by 10. Perhaps taking twice the
round-trip delay time would have been a better idea?
"""
def __init__(self, link):
super().__init__()
self.link = link
self.process_number = self.link.process_number
self.send = self.link.register_abstraction(self)
self.peers = set()
self.detected = set()
self.correct = set()
self.lock = Lock()
self.worker = Thread(target=self.detect_failures)
self.logger = Logging(self.process_number, "PFD")
def start(self):
super().start()
self.worker.start()
def add_peers(self, *peers):
self.peers.update(peers)
def request(self, source_number):
self.logger.log_debug(f"Request from {source_number}")
self.send(source_number, self.reply)
def reply(self, source_number):
self.logger.log_debug(f"Reply from {source_number}")
with self.lock:
self.correct.add(source_number)
def detect_failures(self):
while self.alive:
self.send_heartbeats()
time.sleep(self.TIMEOUT / 10)
self.timeout()
def send_heartbeats(self):
for peer in self.peers - self.detected:
self.send(peer, self.request)
def timeout(self):
with self.lock:
for peer in self.peers - self.correct - self.detected:
self.logger.log_debug(f"Peer {peer} crashed")
self.detected.add(peer)
self.call_callbacks(peer)
self.correct.clear()
