def _handle_primary(self, conn):
_, number, num_requests, checkpoint = utils.recv(conn)
while checkpoint is not None:
self._print(f'Received checkpoint (#{number}) {checkpoint}')
if int(num_requests) > self._num_requests:
with self._lock:
self._state = checkpoint
self._num_requests = int(num_requests)
for request in self._log:
self._state.update(int(request))
self._log = []
try:
utils.send(conn, self._identifier, number, 'ok')
_, number, num_requests, checkpoint = utils.recv(conn)
except Exception:
with self._lock:
if self._primary_index is not None:
self._print('Connection closed by Primary')
self._connected[self._primary_index] = False
self._primary_index = None
time.sleep(1 + random.random() * 5)
self._elect()
return
with self._lock:
if self._primary_index is not None:
self._print('Connection closed by Primary')
self._connected[self._primary_index] = False
self._primary_index = None
time.sleep(1 + random.random() * 5)
self._elect()
def _handle_lfd(self, conn, lfd_identifier):
self._print(f'Connection from LFD {lfd_identifier}')
_, _, message, _ = utils.recv(conn)
while message is not None:
if not self._rm_connected:
self._connect()
if message == 'add':
self._members.append(lfd_identifier)
self._print(f'Added member {lfd_identifier}')
self._print(f'Current members: {self._members}')
if self._rm_connected:
utils.send(self._rm_sock, lfd_identifier, 0, 'add')
elif message == 'remove':
self._members.remove(lfd_identifier)
self._print(f'Removed member {lfd_identifier}')
self._print(f'Current members: {self._members}')
if self._rm_connected:
utils.send(self._rm_sock, lfd_identifier, 0, 'remove')
_, _, message, _ = utils.recv(conn)
self._print(f'Connection closed by LFD {lfd_identifier}')
if lfd_identifier in self._members:
self._members.remove(lfd_identifier)
self._print(f'Removed member {lfd_identifier}')
self._print(f'Current members: {self._members}')
def _handle_lfd(self, conn, lfd_identifier):
self._print(f'Connection from LFD {lfd_identifier}')
_, number, heartbeat, _ = utils.recv(conn)
while heartbeat is not None:
utils.send(conn, self._identifier, number, heartbeat)
_, number, heartbeat, _ = utils.recv(conn)
self._print(f'Connection closed by LFD {lfd_identifier}')
def _heartbeat(self):
# connect to server
connected, server_identifier = self._connect()
# connect to gfd
self._print(f'Connecting to GFD at {self._gfd_hostport}')
self._gfd_sock.connect(utils.address(self._gfd_hostport))
utils.send(self._gfd_sock, self._identifier, 0, 'lfd')
gfd_identifier, _, _, _ = utils.recv(self._gfd_sock)
# make sure gfd is still connected
if gfd_identifier is None:
self._gfd_sock.close()
self._gfd_sock = socket.socket()
self._print('Connection closed by GFD at ' f'{self._gfd_hostport}')
return
self._print(f'Connected to GFD {gfd_identifier}')
member = False
number = 1
while True:
if not connected:
connected, server_identifier = self._connect()
if connected:
self._print(f'Sending heartbeat #{number} to Server '
f'{server_identifier}')
utils.send(self._server_sock, self._identifier, number,
'heartbeat')
_, res_number, response, _ = utils.recv(self._server_sock)
if response is None:
self._print(f'No response from Server {server_identifier}')
self._server_sock.close()
self._server_sock = socket.socket()
connected = False
if member:
self._print(f'Alerting GFD')
utils.send(self._gfd_sock, self._identifier, 0,
'remove')
member = False
else:
self._print(
f'Heartbeat response #{res_number} from Server '
f'{server_identifier}')
if not member:
self._print(f'Registering membership with GFD')
utils.send(self._gfd_sock, self._identifier, 0, 'add')
member = True
number += 1
time.sleep(self._interval)
def _connect(self, index):
try:
sock = self._server_socks[index]
server_hostport = self._server_hostports[index]
sock.connect(utils.address(server_hostport))
utils.send(sock, self._identifier, 0, 'server')
identifier, number, _, state = utils.recv(sock)
# make sure server is still connected
if identifier is None:
sock.close()
self._server_socks[index] = socket.socket()
self._connected[index] = False
else:
self._print(f'Connected to Server {identifier}')
# update state
if number > self._num_requests:
with self._lock:
self._print('Updating state')
self._state = state
self._num_requests = number
if self._log:
self._print('Clearing log')
for request in self._log:
self._state.update(int(request))
self._log = []
self._ready = True
self._ready = True
self._connected[index] = True
except Exception:
self._connected[index] = False
def _handle_gfd(self, conn, gfd_identifier):
self._print(f'Connection from GFD {gfd_identifier}')
identifier, _, message, _ = utils.recv(conn)
while message is not None:
if message == 'add':
self._members.append(identifier)
self._print(f'Added member {identifier}')
self._print(f'Current members: {self._members}')
elif message == 'remove':
self._members.remove(identifier)
self._print(f'Removed member {identifier}')
self._print(f'Current members: {self._members}')
identifier, _, message, _ = utils.recv(conn)
self._print(f'Connection closed by GFD {gfd_identifier}')
for member in self._members:
self._members.remove(member)
self._print(f'Removed member {member}')
self._print(f'Current members: {self._members}')
def _listen(self):
self._print(f'Starting at hostport {self._hostport}')
self._sock.listen()
while True:
conn, _ = self._sock.accept()
identifier, number, data, _ = utils.recv(conn)
utils.send(conn, self._identifier, number, 'rm')
# check for gfd
if data == 'gfd':
Thread(target=self._handle_gfd, args=[conn,
identifier]).start()
def _handle_client(self, conn, client_identifier):
self._print(f'Connection from Client {client_identifier}')
_, number, request, _ = utils.recv(conn)
while request is not None:
self._print(f'Received (#{number}) {request} from Client '
f'{client_identifier}')
with self._lock:
if (not self._ready
or (not self.is_active() and not self.is_primary())):
self._log.append(request)
self._print('Added request to log')
utils.send(conn, self._identifier, number, 'ok')
else:
response = self._state.update(int(request))
self._num_requests += 1
self._print(f'Sending (#{number}) {response} to Client '
f'{client_identifier}')
utils.send(conn, self._identifier, number, response)
_, number, request, _ = utils.recv(conn)
self._print(f'Connection closed by Client {client_identifier}')
def _run_active(self, conn, identifier, number, data):
while True:
# check connection type
if data == 'lfd':
utils.send(conn, self._identifier, number, 'server')
self._handle_lfd(conn, identifier)
return
if data == 'client':
utils.send(conn, self._identifier, number, 'server')
self._handle_client(conn, identifier)
return
if data == 'server':
utils.send(conn, self._identifier, self._num_requests,
'server', self._state)
if data is None:
return
identifier, number, data, _ = utils.recv(conn)
def _connect(self):
try:
self._print(f'Connecting to server at {self._server_hostport}')
self._server_sock.connect(utils.address(self._server_hostport))
utils.send(self._server_sock, self._identifier, 0, 'lfd')
server_identifier, _, _, _ = utils.recv(self._server_sock)
# make sure server is still connected
if server_identifier is None:
self._server_sock.close()
self._server_sock = socket.socket()
self._print('Connection closed by server at '
f'{self._server_hostport}')
return False, server_identifier
self._print(f'Connected to Server {server_identifier}')
return True, server_identifier
except Exception:
return False, None
def _connect(self):
try:
self._print(f'Connecting to RM at {self._rm_hostport}')
self._rm_sock.connect(utils.address(self._rm_hostport))
utils.send(self._rm_sock, self._identifier, 0, 'gfd')
rm_identifier, _, _, _ = utils.recv(self._rm_sock)
# make sure rm is still connected
if rm_identifier is None:
self._rm_sock.close()
self._rm_sock = socket.socket()
self._print('Connection closed by RM at '
f'{self._rm_hostport}')
self._rm_connected = False
self._print(f'Connected to RM {rm_identifier}')
self._rm_connected = True
except Exception:
self._rm_connected = False
def _run_passive(self, conn, identifier, number, data):
while True:
# check connection type
if data == 'lfd':
utils.send(conn, self._identifier, number, 'server')
self._handle_lfd(conn, identifier)
return
if data == 'client':
utils.send(conn, self._identifier, number, 'server')
self._handle_client(conn, identifier)
return
if data == 'server':
utils.send(conn, self._identifier, self._num_requests,
'server', self._state)
elif data == 'elect':
with self._lock:
if not self.is_primary() and self._primary_index is None:
utils.send(conn, self._identifier, number, 'approve')
elif self._primary_index is not None:
utils.send(conn, self._identifier, number,
'disapprove')
else:
utils.send(conn, self._identifier, number,
'primary|' + self._hostport)
elif data is not None and 'primary' in data:
with self._lock:
if self._primary_index is None:
self._print('Primary: ' + identifier)
self._primary = False
self._ready = False
server_hostport = data.split('|')[1]
self._primary_index = self._server_hostports.index(
server_hostport)
utils.send(conn, self._identifier, number, 'backup')
self._handle_primary(conn)
elif data == 'backup':
if self.is_primary():
self._handle_backup(conn, identifier)
return
if data is None:
return
identifier, number, data, _ = utils.recv(conn)
def _listen(self):
self._print(f'Starting at hostport {self._hostport}')
self._sock.listen()
for i, connected in enumerate(self._connected):
if not connected:
self._connect(i)
self._elect()
while True:
for i, connected in enumerate(self._connected):
if not connected:
self._connect(i)
conn, _ = self._sock.accept()
identifier, number, data, _ = utils.recv(conn)
if self.is_active():
Thread(target=self._run_active,
args=[conn, identifier, number, data]).start()
else:
Thread(target=self._run_passive,
args=[conn, identifier, number, data]).start()
def _handle_backup(self, conn, identifier):
number = 1
while True:
self._print(f'Sending checkpoint (#{number}) {self._state} to '
f'Server {identifier}')
try:
utils.send(conn,
self._identifier,
number,
self._num_requests,
state=self._state)
_, _, res, _ = utils.recv(conn)
except Exception:
res = None
if res is None:
self._print(f'Connection closed by Server {identifier}')
return
number += 1
time.sleep(self._interval)
def _elect(self):
for i in range(len(self._server_socks)):
sock = self._server_socks[i]
try:
utils.send(sock, self._identifier, 0, 'elect')
identifier, number, data, _ = utils.recv(sock)
self._lock.acquire()
if self._primary_index is not None:
self._lock.release()
return
if data is not None and 'primary' in data:
self._primary = False
self._ready = False
self._primary_index = i
utils.send(sock, self._identifier, number, 'backup')
self._print('Primary: ' + identifier)
self._lock.release()
Thread(target=self._run_passive,
args=[sock, identifier, number, data]).start()
return
if data == 'approve':
self._primary = True
self._ready = True
self._primary_index = None
utils.send(sock, self._identifier, number,
'primary|' + self._hostport)
self._print('Elected Primary')
self._lock.release()
Thread(target=self._run_passive,
args=[sock, identifier, number, data]).start()
return
self._lock.release()
except Exception:
pass
# no other servers have responded
with self._lock:
self._primary = True
self._ready = True
self._primary_index = None
self._print('Default Primary')