def main():
frontend = "192.168.0.179"
parser = argparse.ArgumentParser(prog="my_megazord_program")
parser.add_argument(
"-f",
"--frontend",
help='start with localshit -f "172.17.0.2" to add frontend')
parser.add_argument(
"-d",
"--debug",
help="start localhost and wait for VS Code debugger",
action="store_true",
)
args = parser.parse_args()
if args.frontend:
frontend = args.frontend
if args.debug:
import debugpy
debugpy.listen(5678)
print("Waiting for debugger attach: %s" % os.getpid())
debugpy.wait_for_client()
logging.info("Frontend server is set to %s" % frontend)
try:
logging.info("starting manager...")
_ = LocalsHitManager(frontend=frontend)
except Exception as e:
logging.error("Error while starting app: %s" % e)
traceback.print_exc()
def work_func(self):
if self.election.isLeader:
# only run content service if elected as leader
if self.server.isRunning is False:
# check if WebsocketServer is running
try:
self.server.run_forever()
self.server.isRunning = True
except Exception as e:
logging.error("Error while restarting server: %s" % e)
else:
if config["chuck_norris"] is True:
# optionally publish chuck norris quotes in a intervall
time_diff = time.time() - self.last_update
if time_diff >= config["quote_intervall"]:
logging.info("Content: publish new quote")
id, quote = self._get_quote("jokes.json")
data = "%s:%s:%s" % ("CO", id, quote)
self._replicate_and_send(data)
self.last_update = time.time()
else:
if self.server.isRunning is True:
self.server.isRunning = False
data = "%s:%s" % ("CL", "close server")
self.server.send_message_to_all(data)
self.server.shutdown()
self.server.server_close()
logging.info("Content: publish service stopped")
def handle_heartbeat_message(self, addr, parts):
# forward heartbeat message as it is, if not leader
if self.election.isLeader is False:
# check, if the heartbeat comes from the neighbour
left_neighbour = self.hosts.get_neighbour(direction="left")
right_neighbour = self.hosts.get_neighbour(direction="right")
# cehck, if heartbeat comes from the right neighbour
if addr[0] == right_neighbour:
# forward message
logging.info("Heartbeat: received. forward to %s" %
left_neighbour)
new_message = "HB:%s:%s" % (parts[1], parts[2])
self.socket_sender.send_message(new_message,
left_neighbour,
type="unicast")
# note time of last heartbeat
self.last_heartbeat_received = time.time()
else:
logging.error("Heartbeat: received from wrong neighbour")
else:
# if leader, have a look at the message if it is from himself
if self.heartbeat_message:
if parts[1] == self.heartbeat_message["id"]:
logging.info("Heartbeat: received own heartbeat from %s." %
addr[0])
self.heartbeat_message = None
self.last_heartbeat_received = time.time()
def watch_heartbeat(self):
# check, when was the last heartbeat from the left neighbour?
time_diff = time.time() - self.last_heartbeat_received
if time_diff >= config["heartbeat_timeout"]:
failed_neighbour = self.hosts.get_neighbour(direction="right")
# if own address, then do nothing
if failed_neighbour is not self.own_address:
# remove failed neighbour
logging.info("Heartbeat: nothing received from %s" %
failed_neighbour)
self.hosts.remove_host(failed_neighbour)
# send failure message as multicast
new_message = "FF:%s:%s" % (failed_neighbour, self.own_address)
self.socket_sender.send_message(new_message, type="multicast")
# if this was leader, then start service announcement and leader election
if failed_neighbour == self.election.elected_leader:
data = "%s:%s" % ("SA", self.own_address)
self.socket_sender.send_message(data, type="multicast")
time.sleep(1)
self.election.start_election(await_response=True)
self.last_heartbeat_received = time.time()
def announce_service(self, timeout=1):
data = "%s:%s" % ("SA", self.own_address)
self.socket_sender.send_message(data, type="multicast")
logging.debug("SA: service announcement...")
time.sleep(timeout)
logging.info("SA: service announcement finished.")
logging.info("Discovered hosts: %s" % self.hosts.sorted_ring)
def _wait_for_response(self, timeout):
# wait a leader is elected or timeout is over
last_response = time.time()
while self.elected_leader == "":
if time.time() - last_response > timeout:
logging.info("Leader Election: No response within timeout.")
self.elected_leader = self.current_member_ip
self.isLeader = True
self.send_election_to_frontend()
break
time.sleep(0.1)
def __init__(self, socket_sender, hosts, frontend):
# first, mark member as non-participant
self.participant = False
self.socket_sender = socket_sender
self.hosts = hosts
self.current_member_ip = self.hosts.current_member_ip
self.frontend_address = frontend
self.isLeader = False
self.got_response = False
self.elected_leader = ""
self.CONTENT_PORT = config.config["frontend_unicast_port"]
logging.info("Election Class initialized")
def start_election(self, await_response=False, timeout=1):
logging.info("starting election")
# mark self as a participant
self.participant = True
# send message to left neighbour
message = "SE:%s:%s" % (self.current_member_ip, self.isLeader)
neighbour = self.hosts.get_neighbour()
self.socket_sender.send_message(message, neighbour, type="unicast")
if await_response is True:
self._wait_for_response(timeout)
def message_received(self, client, server, message):
"""Callback function of WebsocketServer on message received from client"""
logging.info("Content: Message from client %d: %s" %
(client["id"], message))
parts = message.split(":")
if parts[0] == "CR":
new_message = "%s:%s:%s" % (parts[0], parts[1], parts[2])
self._replicate_and_send(new_message)
# self.server.send_message_to_all(new_message)
elif parts[0] == "CO":
quote_id = uuid.uuid4()
data = "%s:%s:%s" % ("CO", quote_id, parts[1])
self._replicate_and_send(data)
def run(self):
""" Initialize and start all threads. """
thread_routines = [
self.ack_handler,
self.message_queue_handler,
self.incoming_message_handler,
]
count = 1
for thread_routine in thread_routines:
thread = threading.Thread(
target=thread_routine,
args=(self.running,),
name="ReliableSocketWorker-%s" % count,
)
thread.daemon = True
thread.start()
logging.info("Thread %s started." % thread.name)
self.threads.append(thread)
count += 1
def send_heartbeat(self):
# create heartbeat message and send it every 3 sec.
time_diff = time.time() - self.last_heartbeat_sent
if time_diff >= config["heartbeat_intervall"]:
self.heartbeat_message = {
"id": str(uuid.uuid4()),
"sender": self.own_address,
"timestamp": time.time(),
}
new_message = "HB:%s:%s" % (
self.heartbeat_message["id"],
self.heartbeat_message["sender"],
)
self.socket_sender.send_message(new_message,
self.hosts.get_neighbour(),
type="unicast")
logging.info("Heartbeat: send to %s" % self.hosts.get_neighbour())
self.last_heartbeat_sent = time.time()
def form_ring(self, own_ip):
self.sorted_ring = self._form_ring(self.members)
logging.info("Discovered hosts: %s" % self.sorted_ring)
left_member = self.get_neighbour(direction="left")
logging.info("Own IP: %s | left Neighbour: %s" % (own_ip, left_member))
right_member = self.get_neighbour(direction="right")
logging.info("Own IP: %s | right Neighbour: %s" % (own_ip, right_member))
def forward_election_message(self, message):
compare = utils.compare_adresses(message[1], self.current_member_ip)
sender_id = message[1]
leader_elected = eval(message[2])
if leader_elected is False:
if compare is CompareResult.LARGER:
# 4.1 if id is larger, forward message to next member
logging.info("Leader Election: Forward message as it is.")
self.participant = True
new_message = "SE:%s:%s" % (sender_id, False)
self.socket_sender.send_message(new_message,
self.hosts.get_neighbour(),
type="unicast")
elif compare is CompareResult.LOWER and self.participant is False:
# 4.2 if id is smaller and not yes marked as participant, replace id and forward message to next member.
self.participant = True
logging.info("Leader Election: Forward message with own id.")
new_message = "SE:%s:%s" % (self.current_member_ip, False)
self.socket_sender.send_message(new_message,
self.hosts.get_neighbour(),
type="unicast")
elif compare is CompareResult.LOWER and self.participant is True:
# 4.3 if id is smaller but already participant, then discard message
logging.info(
"Leader Election: Already participant of an election. Discard message."
)
elif compare is CompareResult.SAME:
# 4.4 if message came back, set itself as leader and start second part algorithm. Inform others about elected leader.
logging.info(
"Leader Election: Message came back to sender. Elected as leader."
)
self.participant = False
self.isLeader = True
self.elected_leader = self.current_member_ip
# start second part of algorithm, inform others about election
new_message = "SE:%s:%s" % (self.current_member_ip, True)
self.socket_sender.send_message(new_message,
self.hosts.get_neighbour(),
type="unicast")
self.send_election_to_frontend()
else:
logging.error("Leader Election: invalid result")
else:
# forward message about elected leader
if sender_id is self.current_member_ip:
logging.info(
"Leader Election: Message came back to sender. Election is over. Elected Leader: %s"
% self.elected_leader)
self.send_election_to_frontend()
elif self.participant is True:
# elected message received. mark as non-participant, record election and forward message
self.participant = False
self.isLeader = False
self.elected_leader = sender_id
logging.info(
"Leader Election: Forward election message. Elected Leader: %s"
% self.elected_leader)
new_message = "SE:%s:%s" % (message[1], message[2])
self.socket_sender.send_message(new_message,
self.hosts.get_neighbour(),
type="unicast")
else:
logging.info(
"Leader Election: Election is over. Elected Leader: %s" %
self.elected_leader)
self.send_election_to_frontend()
def __init__(self, frontend=None):
self.threads = []
if frontend is None:
self.frontend = config["frontend_server"]
else:
self.frontend = frontend
self.running = True
self.isActive = False
logging.info("manager started!")
# init socket connections
self.socket_sender = SocketSender()
self.own_address = utils.get_host_address()
# init Ring
self.hosts = Ring(self.own_address)
# init service announcement object
self.service_announcement = ServiceAnnouncement(
self.hosts, self.socket_sender)
# init election
self.election = Election(self.socket_sender,
self.hosts,
frontend=frontend)
# init database
self.database = Database()
# init ReliableSocketWorker
self.reliable_socket = ReliableSocketWorker(
self.running, self.hosts, port=config["reliable_socket"])
try:
self.reliable_socket.run()
self.heartbeat = Heartbeat(self.hosts, self.election,
self.socket_sender)
# initiate service discovery thread
self.discovery_thread = ServiceDiscovery(
self.service_announcement,
self.hosts,
self.election,
self.heartbeat,
self.isActive,
)
self.discovery_thread.start()
self.threads.append(self.discovery_thread)
# start service announcement after discovery
self.service_announcement.announce_service(
timeout=config["announcement_timeout"])
# start election after discovery
self.election.start_election(await_response=True, timeout=1)
self.discovery_thread.isActive = True
# initiate Content Provider
content_provider = ContentProvider(self.election,
self.reliable_socket,
self.database)
content_provider.start()
self.threads.append(content_provider)
# monitor threads and exit on failing
while self.running:
for th in self.threads:
if not th.is_alive():
logging.info("Thread %s died." % th.__class__.__name__)
self.running = False
break
time.sleep(0.2)
except KeyboardInterrupt:
logging.info("Process terminated by user")
except Exception as e:
logging.error("Error in run.py: %s" % e)
traceback.print_exc()
finally:
# graceful shutdown
self.isActive = False
self.discovery_thread.isActive = False
logging.info("stopping threads...")
for th in self.threads:
logging.info("Stopping thread %s." % th.__class__.__name__)
th.stop()
for th in self.threads:
logging.info("Joining thread %s." % th.__class__.__name__)
th.join()
for thread in self.reliable_socket.threads:
logging.info("Joining Thread %s." % thread.name)
thread.join(0.2)
main_thread = threading.currentThread()
for t in threading.enumerate():
if t is main_thread:
continue
logging.info("Joining Thread %s." % thread.name)
t.join(0.2)
logging.info("threads stopped")
def client_left(self, client, server):
"""Callback function of WebsocketServer on client left"""
logging.info("Content: Client(%d) disconnected" % client["id"])
def new_client(self, client, server):
"""Callback function of WebsocketServer on new client connected"""
logging.info("Content: New client connected and was given id %d" %
client["id"])