def handle_heartbeat_message(self, addr, parts):
# forward heartbeat message as it is, if not leader
if self.election.isLeader is False:
# check, if the heartbeat comes from the neighbour
left_neighbour = self.hosts.get_neighbour(direction="left")
right_neighbour = self.hosts.get_neighbour(direction="right")
# cehck, if heartbeat comes from the right neighbour
if addr[0] == right_neighbour:
# forward message
logging.info("Heartbeat: received. forward to %s" %
left_neighbour)
new_message = "HB:%s:%s" % (parts[1], parts[2])
self.socket_sender.send_message(new_message,
left_neighbour,
type="unicast")
# note time of last heartbeat
self.last_heartbeat_received = time.time()
else:
logging.error("Heartbeat: received from wrong neighbour")
else:
# if leader, have a look at the message if it is from himself
if self.heartbeat_message:
if parts[1] == self.heartbeat_message["id"]:
logging.info("Heartbeat: received own heartbeat from %s." %
addr[0])
self.heartbeat_message = None
self.last_heartbeat_received = time.time()
def main():
frontend = "192.168.0.179"
parser = argparse.ArgumentParser(prog="my_megazord_program")
parser.add_argument(
"-f",
"--frontend",
help='start with localshit -f "172.17.0.2" to add frontend')
parser.add_argument(
"-d",
"--debug",
help="start localhost and wait for VS Code debugger",
action="store_true",
)
args = parser.parse_args()
if args.frontend:
frontend = args.frontend
if args.debug:
import debugpy
debugpy.listen(5678)
print("Waiting for debugger attach: %s" % os.getpid())
debugpy.wait_for_client()
logging.info("Frontend server is set to %s" % frontend)
try:
logging.info("starting manager...")
_ = LocalsHitManager(frontend=frontend)
except Exception as e:
logging.error("Error while starting app: %s" % e)
traceback.print_exc()
def work_func(self):
if self.election.isLeader:
# only run content service if elected as leader
if self.server.isRunning is False:
# check if WebsocketServer is running
try:
self.server.run_forever()
self.server.isRunning = True
except Exception as e:
logging.error("Error while restarting server: %s" % e)
else:
if config["chuck_norris"] is True:
# optionally publish chuck norris quotes in a intervall
time_diff = time.time() - self.last_update
if time_diff >= config["quote_intervall"]:
logging.info("Content: publish new quote")
id, quote = self._get_quote("jokes.json")
data = "%s:%s:%s" % ("CO", id, quote)
self._replicate_and_send(data)
self.last_update = time.time()
else:
if self.server.isRunning is True:
self.server.isRunning = False
data = "%s:%s" % ("CL", "close server")
self.server.send_message_to_all(data)
self.server.shutdown()
self.server.server_close()
logging.info("Content: publish service stopped")
def _ask_for_initial_database(self):
data = "%s:%s" % ("AA", self.election.current_member_ip)
try:
self.reliable_socket.multicast(data)
except Exception as e:
logging.error("Content: Error while saving quote: %s" % e)
return
def _send_unicast(self, message, address, port):
if port is None:
port = self.UCAST_PORT
if address is None:
logging.error("No address given. Cannot send message.")
return
self.socket_unicast.sendto(message.encode(), (address, port))
def get_range(self, start=None, end=None):
try:
if start and end:
return self.database[start:end]
elif start:
return self.database[start:]
elif end:
return self.database[:end]
else:
return None
except Exception as e:
logging.error("Error at get_range: %s" % e)
def _replicate_and_send(self, data):
# 1. replicate with other backend servers and itself to store quote to database
try:
self.reliable_socket.multicast(data)
except Exception as e:
logging.error("Content: Error while saving quote: %s" % e)
return
# 2. Send message to client
try:
self.server.send_message_to_all(data)
except Exception as e:
logging.error("Content: Error while sending quote: %s" % e)
return
def _get_quote(self, filename):
quote = None
quote_id = None
try:
file = open(filename)
data = json.load(file)
quotes = data["value"]
counts = len(quotes)
rand = random.randint(0, counts - 1)
quote = quotes[rand]
quote = quote["joke"]
quote_id = uuid.uuid4()
except Exception as e:
logging.error("Content: Error while generating quote: %s" % e)
return (quote_id, quote)
def work_func(self):
"""
Manages all incoming messages on unicast and multicast socket
"""
try:
# listen to incoming messages on multicast and unicast
inputready, outputready, exceptready = select(
[self.socket_multicast, self.socket_unicast], [], [], 1,
)
for socket_data in inputready:
# handle UDP socket connections
data, addr = socket_data.recvfrom(1024) # wait for a packet
if data:
parts = data.decode().split(":")
if parts[0] == "SA":
self.service_announcement.handle_service_announcement(addr)
elif parts[0] == "SE":
self.election.forward_election_message(parts)
elif parts[0] == "HB":
# TODO: forward heartbeat
self.heartbeat.handle_heartbeat_message(addr, parts)
elif parts[0] == "FF":
self.heartbeat.handle_failure_message(addr, parts)
elif parts[0] == "RP":
logging.error("Reply from host: %s" % addr[0])
if addr[0] != utils.get_host_address():
self.hosts.add_host(addr[0])
else:
logging.error("Unknown message type: %s" % parts[0])
# reset heartbeat beacause of leader election or service announcement
self.heartbeat.last_heartbeat_received = time.time()
except Exception as e:
logging.error("Error: %s" % e)
# check if announcement and election is over:
if self.isActive is True:
# send heartbeat messages
if self.election.isLeader is True:
self.heartbeat.send_heartbeat()
# watch heartbeats
th = threading.Thread(target=self.heartbeat.watch_heartbeat)
th.start()
def forward_election_message(self, message):
compare = utils.compare_adresses(message[1], self.current_member_ip)
sender_id = message[1]
leader_elected = eval(message[2])
if leader_elected is False:
if compare is CompareResult.LARGER:
# 4.1 if id is larger, forward message to next member
logging.info("Leader Election: Forward message as it is.")
self.participant = True
new_message = "SE:%s:%s" % (sender_id, False)
self.socket_sender.send_message(new_message,
self.hosts.get_neighbour(),
type="unicast")
elif compare is CompareResult.LOWER and self.participant is False:
# 4.2 if id is smaller and not yes marked as participant, replace id and forward message to next member.
self.participant = True
logging.info("Leader Election: Forward message with own id.")
new_message = "SE:%s:%s" % (self.current_member_ip, False)
self.socket_sender.send_message(new_message,
self.hosts.get_neighbour(),
type="unicast")
elif compare is CompareResult.LOWER and self.participant is True:
# 4.3 if id is smaller but already participant, then discard message
logging.info(
"Leader Election: Already participant of an election. Discard message."
)
elif compare is CompareResult.SAME:
# 4.4 if message came back, set itself as leader and start second part algorithm. Inform others about elected leader.
logging.info(
"Leader Election: Message came back to sender. Elected as leader."
)
self.participant = False
self.isLeader = True
self.elected_leader = self.current_member_ip
# start second part of algorithm, inform others about election
new_message = "SE:%s:%s" % (self.current_member_ip, True)
self.socket_sender.send_message(new_message,
self.hosts.get_neighbour(),
type="unicast")
self.send_election_to_frontend()
else:
logging.error("Leader Election: invalid result")
else:
# forward message about elected leader
if sender_id is self.current_member_ip:
logging.info(
"Leader Election: Message came back to sender. Election is over. Elected Leader: %s"
% self.elected_leader)
self.send_election_to_frontend()
elif self.participant is True:
# elected message received. mark as non-participant, record election and forward message
self.participant = False
self.isLeader = False
self.elected_leader = sender_id
logging.info(
"Leader Election: Forward election message. Elected Leader: %s"
% self.elected_leader)
new_message = "SE:%s:%s" % (message[1], message[2])
self.socket_sender.send_message(new_message,
self.hosts.get_neighbour(),
type="unicast")
else:
logging.info(
"Leader Election: Election is over. Elected Leader: %s" %
self.elected_leader)
self.send_election_to_frontend()
def __init__(self, frontend=None):
self.threads = []
if frontend is None:
self.frontend = config["frontend_server"]
else:
self.frontend = frontend
self.running = True
self.isActive = False
logging.info("manager started!")
# init socket connections
self.socket_sender = SocketSender()
self.own_address = utils.get_host_address()
# init Ring
self.hosts = Ring(self.own_address)
# init service announcement object
self.service_announcement = ServiceAnnouncement(
self.hosts, self.socket_sender)
# init election
self.election = Election(self.socket_sender,
self.hosts,
frontend=frontend)
# init database
self.database = Database()
# init ReliableSocketWorker
self.reliable_socket = ReliableSocketWorker(
self.running, self.hosts, port=config["reliable_socket"])
try:
self.reliable_socket.run()
self.heartbeat = Heartbeat(self.hosts, self.election,
self.socket_sender)
# initiate service discovery thread
self.discovery_thread = ServiceDiscovery(
self.service_announcement,
self.hosts,
self.election,
self.heartbeat,
self.isActive,
)
self.discovery_thread.start()
self.threads.append(self.discovery_thread)
# start service announcement after discovery
self.service_announcement.announce_service(
timeout=config["announcement_timeout"])
# start election after discovery
self.election.start_election(await_response=True, timeout=1)
self.discovery_thread.isActive = True
# initiate Content Provider
content_provider = ContentProvider(self.election,
self.reliable_socket,
self.database)
content_provider.start()
self.threads.append(content_provider)
# monitor threads and exit on failing
while self.running:
for th in self.threads:
if not th.is_alive():
logging.info("Thread %s died." % th.__class__.__name__)
self.running = False
break
time.sleep(0.2)
except KeyboardInterrupt:
logging.info("Process terminated by user")
except Exception as e:
logging.error("Error in run.py: %s" % e)
traceback.print_exc()
finally:
# graceful shutdown
self.isActive = False
self.discovery_thread.isActive = False
logging.info("stopping threads...")
for th in self.threads:
logging.info("Stopping thread %s." % th.__class__.__name__)
th.stop()
for th in self.threads:
logging.info("Joining thread %s." % th.__class__.__name__)
th.join()
for thread in self.reliable_socket.threads:
logging.info("Joining Thread %s." % thread.name)
thread.join(0.2)
main_thread = threading.currentThread()
for t in threading.enumerate():
if t is main_thread:
continue
logging.info("Joining Thread %s." % thread.name)
t.join(0.2)
logging.info("threads stopped")