def __init__(self, is_leader, leader_hostname, my_hostname, tcp_port=13337, sloppy_Qsize=5, sloppy_R=3, sloppy_W=3):

self.ongoing_requests = []

self.is_leader = is_leader

self.leader_hostname = leader_hostname

self.hostname = my_hostname

self.tcp_port = tcp_port

self.my_address = (self.hostname, self.tcp_port)

self.membership_ring = Ring(replica_count=sloppy_Qsize - 1) # Other nodes in the membership

if self.is_leader:

self.membership_ring.add_node(leader_hostname)

# todo: look into this, do we need both?

self.bootstrapping = True

self.is_member = False

# Flag to keep track of a add-node is underway

self._membership_in_progress = False

self.sloppy_Qsize = sloppy_Qsize # total members to replicate on

# number of peers required for a read or write to succeed.

self.sloppy_R = sloppy_R

self.sloppy_W = sloppy_W

# Book keeping for membership messages

self._req_responses = defaultdict(set)

self._sent_req_messages = {}

self._received_req_messages = {}

self._req_sender = {} # Keeps track to sender for add and delete requests

self.current_view = 0 # increment this on every leader election

self.membership_request_id = 0 # increment this on every request sent to peers

# Maintains handoff messages to be sent

# IP : set(handoff messages)

self._handoff_messages = defaultdict(set)

self.handoff_timer = None

self.create_handoff_timer = lambda: Timer(5, self.try_sending_handoffs)

self.log_prefix = os.getcwd()

self.ring_log_file = os.path.join(self.log_prefix, self.hostname + '.ring')

self.db_path = os.path.join(self.log_prefix, self.hostname + '.db')

self.handoff_log = os.path.join(self.log_prefix, self.hostname + '.pickle')

try:

with open(self.ring_log_file, 'r') as f:

hosts = f.readlines()

for h in hosts:

self.membership_ring.add_node(h.strip())

print("Restored membership information from %s" % self.ring_log_file)

except FileNotFoundError:

pass

try:

with open(self.handoff_log, 'rb') as f:

self._handoff_messages = pickle.loads(f.read())

if len(self._handoff_messages) > 0:

self.handoff_timer = self.create_handoff_timer()

self.handoff_timer.start()

print("Restored hand off messages from %s" % self.handoff_log)

except FileNotFoundError:

pass

self.request_timelimit = 2.0

self.req_message_timers = {}

self.db = Storage(self.db_path) # set up sqlite table

# create tcp socket for communication with peers and clients

self.tcp_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)

self.tcp_socket.setblocking(False) # Non-blocking socket

self.tcp_socket.bind((self.hostname, self.tcp_port))

self.tcp_socket.listen(10)

# has hostnames mapped to open sockets

self.connections = {}

self.client_list = set()

def accept_connections(self):

incoming_connections = {self.tcp_socket}

print("Accepting connections...")

while True:

readable, _, _ = select.select(incoming_connections, [], [], 0)

for s in readable:

if s is self.tcp_socket:

connection, client_address = s.accept()

connection.setblocking(False)

incoming_connections.add(connection)

self.connections[client_address[0]] = connection

else:

try:

header = s.recv(5)

except:

print("Connection reset.")

continue

if not header: # remove for connection pool and close socket

incoming_connections.remove(s)

# del self.connections[s.getpeername()[0]]

s.close()

else:

message_len = struct.unpack('!i', header[1:5])[0]

data = b''

while len(data) < message_len:

try:

data += s.recv(message_len - len(data))

except socket.error as err:

pass

self._process_message(header + data, s.getpeername()[0]) # addr is a tuple of hostname and port

def _process_message(self, data, sender):

message_type, data_tuple = messages._unpack_message(data)

message_type_mapping = {

b'\x00': self._process_command,

b'\x01': self._process_req_message,

b'\x10': self._membership_change_message,

b'\xff': self._process_ok_message,

b'\x07': self.perform_operation,

b'\x08': self.perform_operation,

b'\x70': self.update_request,

b'\x80': self.update_request,

b'\x0B': self.update_request,

b'\x0A': self.handle_forwarded_req,

b'\x0C': self.handle_handoff

}

message_type_mapping[message_type](data_tuple, sender)

return

def _process_command(self, user_input, sendBackTo):

"""Process commands"""

self.client_list.add(sendBackTo)

# Maps command to the corresponding function.

# Command arguments are passed as the first argument to the function.

command_registry = { # Possible commands:

"add-node": self.add_node, # 1. add node to membership

"remove-node": self.remove_node, # 2. remove node from membership

"put": self.put_data, # 3. put data

"get": self.get_data, # 4. get data

}

if not user_input:

self._send_req_response_to_client(sendBackTo, "User input empty")

# First word is command. Rest are then arguments.

command, *data = user_input.split(" ")

if command not in command_registry:

self._send_req_response_to_client(sendBackTo, "Invalid command")

# Call the function associated with the command in command_registry

return command_registry[command](data, sendBackTo)

def handle_handoff(self, data, sendBackTo):

# data should have (message, list of hosts to hand data off to)

for h in data[1]:

print("Storing hand off message for %s" % h)

self._handoff_messages[h].add(data[0])

# Save handoff messages to disk

self.sync_handoffs_to_disk()

# check and start timer

if not self.handoff_timer:

self.handoff_timer = self.create_handoff_timer()

self.handoff_timer.start()

def try_sending_handoffs(self):

print("Attempting to send hand off messages...")

updated_handoff_messages = defaultdict(set)

for (host, msgs) in self._handoff_messages.items():

for msg in msgs:

fail = self.broadcast_message([host], msg)

if fail:

updated_handoff_messages[host].add(msg)

self._handoff_messages = updated_handoff_messages

self.sync_handoffs_to_disk()

if len(self._handoff_messages) > 0:

print("Undelivered hand offs. Restarting timer")

self.handoff_timer = self.create_handoff_timer()

self.handoff_timer.start()

return

print("Sent all hand off messages.")

self.handoff_timer = None

def sync_handoffs_to_disk(self):

# Save hand off messages to disk

with open(self.handoff_log, 'wb') as f:

f.write(pickle.dumps(self._handoff_messages))

def add_node(self, data, sender):

"""Add node to membership. data[0] must be the hostname. Initiates 2PC."""

if not self.is_leader:

self._send_req_response_to_client(sender, "Error: This is not the leader")

return

if not data:

self._send_req_response_to_client(sender, "Error: hostname required")

return

if self._membership_in_progress:

self._send_req_response_to_client(sender, "Membership operation in progress. Try again")

return

if data[0] in self.membership_ring:

self._send_req_response_to_client(sender, "Already in the membership")

return

self._membership_in_progress = True

print("Starting add-node operation for %s" % data[0])

new_peer_message = messages.reqMessage(self.current_view, self.membership_request_id, 1, data[0])

# associate hostname to (view_id, req_id)

self._sent_req_messages[(self.current_view, self.membership_request_id)] = (data[0], 1)

self._req_sender[(self.current_view, self.membership_request_id)] = sender

# broadcast to all but leader.

nodes_to_broadcast = self.membership_ring.get_all_hosts()

nodes_to_broadcast.remove(self.hostname)

nodes_to_broadcast.add(data[0]) # Add new host to broadcast list

self.broadcast_message(nodes_to_broadcast, new_peer_message)

t = Timer(self.request_timelimit, self._req_timeout, args=[(self.current_view, self.membership_request_id)])

self.req_message_timers[(self.current_view, self.membership_request_id)] = t

t.start()

self.membership_request_id += 1

# Send a remove node message to everyone and if you are that node, shutdown

def remove_node(self, data, sender):

if not self.is_leader:

self._send_req_response_to_client(sender, "Error: This is not the leader")

return

if not data:

self._send_req_response_to_client(sender, "Error: hostname required")

return

if self._membership_in_progress:

self._send_req_response_to_client(sender, "Membership operation in progress. Try again")

return

if data[0] not in self.membership_ring:

self._send_req_response_to_client(sender, "Cannot remove. Node not in membership")

return

self._membership_in_progress = True

print("Starting remove-node operation for %s" % data[0])

new_peer_message = messages.reqMessage(self.current_view, self.membership_request_id, 2, data[0])

# associate hostname to (view_id, req_id)

self._sent_req_messages[(self.current_view, self.membership_request_id)] = (data[0], 2)

self._req_sender[(self.current_view, self.membership_request_id)] = sender

# broadcast to all but leader.

nodes_to_broadcast = self.membership_ring.get_all_hosts()

nodes_to_broadcast.remove(self.hostname)

nodes_to_broadcast.add(data[0]) # Add new host to broadcast list

self.broadcast_message(nodes_to_broadcast, new_peer_message)

t = Timer(self.request_timelimit, self._req_timeout, args=[(self.current_view, self.membership_request_id)])

self.req_message_timers[(self.current_view, self.membership_request_id)] = t

t.start()

self.membership_request_id += 1

# self.membership_ring.remove_node(data[0])

def put_data(self, data, sendBackTo):

if len(data) != 3:

return "Error: Invalid operands\nInput: (<key>,<prev version>,<value>)"

data = [data[0], json.loads(data[1]), data[2]]

key = data[0]

prev = data[1]

value = data[2]

target_node = self.membership_ring.get_node_for_key(data[0])

if not self.is_leader:

# forward request to leader for client

return self._send_data_to_peer(self.leader_hostname, data, sendBackTo)

else: # I am the leader

if target_node == self.hostname:

# I'm processing a request for a client directly

self.start_request('put', data, sendBackTo=sendBackTo)

return

else: # I am forwarding a request from the client to the correct node

return self._send_data_to_peer(target_node, data, sendBackTo)

def get_data(self, data, sendBackTo):

"""Retrieve V for given K from the database. data[0] must be the key"""

if not data:

return "Error: key required"

target_node = self.membership_ring.get_node_for_key(data[0])

# if I can do it myself

if target_node == self.hostname:

# I am processing a request for a client directly

self.start_request('get', data[0], sendBackTo=sendBackTo)

else: # forward the client request to the peer incharge of req

self._request_data_from_peer(target_node, data[0], sendBackTo)

def _process_req_message(self, data, sender):

# data = (view_id, req_id, operation, address)

(view_id, req_id, operation, address) = data

print("Processed request message to add %s" % address)

# save the message type

self._received_req_messages[(view_id, req_id)] = (address, operation)

ok_message = messages.okMessage(view_id, req_id)

self.connections.get(sender, self._create_socket(sender)).sendall(ok_message)

def _process_ok_message(self, data, sender):

self._req_responses[data].add(sender)

(new_peer_hostname, operation) = self._sent_req_messages[data]

required_responses = len(self.membership_ring) if operation == 1 else (len(self.membership_ring) - 1)

# number of replies equal number of *followers* already in the ring, add peer to membership

if len(self._req_responses[data]) == required_responses:

# Cancel timer

t = self.req_message_timers.get(data, None)

if not t:

return

t.cancel()

# Send newViewMessage

# self.current_view += 1

if operation == 1: # adding new node

self.membership_ring.add_node(new_peer_hostname)

hosts_to_send = self.membership_ring.get_all_hosts()

nodes_to_broadcast = self.membership_ring.get_all_hosts()

else:

hosts_to_send = (new_peer_hostname, )

nodes_to_broadcast = self.membership_ring.get_all_hosts()

nodes_to_broadcast.remove(new_peer_hostname)

self.membership_ring.remove_node(new_peer_hostname)

membership_change_msg = messages.membershipChange(self.current_view, operation, hosts_to_send)

nodes_to_broadcast.remove(self.hostname)

self.broadcast_message(nodes_to_broadcast, membership_change_msg)

print("Successfully %s %s." % ("added" if operation == 1 else "removed", new_peer_hostname))

client = self._req_sender.get(data, None)

self._send_req_response_to_client(client, "Successfully %s %s." %

("added" if operation == 1 else "removed", new_peer_hostname))

self._membership_in_progress = False # reset state to accept new connections

def _membership_change_message(self, data, sender):

(view_id, operation, peers) = data

# self.current_view = view_id

if operation == 1: # add nodes

for p in peers:

if p not in self.membership_ring:

self.membership_ring.add_node(p)

if operation == 2:

for p in peers:

if p in self.membership_ring:

self.membership_ring.remove_node(p)

with open(self.ring_log_file, 'w') as f:

for node in self.membership_ring.get_all_hosts():

f.write(node + '\n')

print("Successfully modified membership ring. Total members: %d" % len(self.membership_ring.get_all_hosts()))

print("Current members: %s" % ", ".join(self.membership_ring.get_all_hosts()))

print("Keys to manage: ", self.membership_ring.get_key_range(self.hostname))

def _req_timeout(self, req_id):

print("Error adding node to network. One or more nodes is offline.")

# failed_hosts = set(self.membership_ring.get_all_hosts()) - self._req_responses[req_id]

# failed_hosts.remove()

sender = self._req_sender.get(req_id, None)

self._send_req_response_to_client(sender, "Failed to add node to the network")

self._membership_in_progress = False

def _send_req_response_to_client(self, client, message):

msg = messages.responseForForward(message)

self.broadcast_message([client], msg)

# request format:

# object which contains

# type

# sendBackTo

# forwardedTo =None if type is not for_*

# hash

# value =None if type is get or forget

# context =None if type is get or forget

# responses = { sender:msg, sender2:msg2... }

# args format is determined by type:

# type='get', args='hash'

# type='put', args=('hash','value',{context})

# type='for_get', args=(target_node,'hash')

# type='for_put', args=(target_node, 'hash','value',{context})

# Type can be 'put', 'get', 'for_put', 'for_get'

# 'for_*' if for requests that must be handled by a different peer

# then when the response is returned, complete_request will send the

# output to the correct client or peer (or stdin)

def start_request(self, rtype, args, sendBackTo, prev_req=None):

print("%s request from %s: %s" % (rtype, sendBackTo, args))

req = Request(rtype, args, sendBackTo, previous_request=prev_req) # create request obj

self.ongoing_requests.append(req) # set as ongoing

target_node = self.membership_ring.get_node_for_key(req.hash)

replica_nodes = self.membership_ring.get_replicas_for_key(req.hash)

T = Timer(self.request_timelimit + (1 if rtype[:3] == 'for' else 0),

self.complete_request, args=[req], kwargs={"timer_expired": True}

)

T.start()

self.req_message_timers[req.time_created] = T

# Find out if you can respond to this request

if rtype == 'get':

# add my information to the request

result = self.db.getFile(args)

my_resp = messages.getFileResponse(args, result, req.time_created)

self.update_request(messages._unpack_message(my_resp)[1], socket.gethostbyname(self.hostname), req)

# send the getFile message to everyone in the replication range

msg = messages.getFile(req.hash, req.time_created)

# this function will need to handle hinted handoff

print("Sending getFile message to %s" % ", ".join(replica_nodes))

fails = self.broadcast_message(replica_nodes, msg)

if fails:

print("Failed to send get msg to %s" % ', '.join(fails))

elif rtype == 'put':

self.db.storeFile(args[0], socket.gethostbyname(self.hostname), args[1], args[2])

my_resp = messages.storeFileResponse(args[0], args[1], args[2], req.time_created)

# add my information to the request

self.update_request(messages._unpack_message(my_resp)[1], socket.gethostbyname(self.hostname), req)

# send the storeFile message to everyone in the replication range

msg = messages.storeFile(req.hash, req.value, req.context, req.time_created)

# this function will need to handle hinted handoff

print("Sending storeFile message to %s" % ", ".join(replica_nodes))

fails = self.broadcast_message(replica_nodes, msg)

if fails:

print("Failed to send put msg to %s" % ', '.join(fails))

else:

msg = messages.forwardedReq(req)

# forward message to target node

# self.connections[req.forwardedTo].sendall(msg)

if self.broadcast_message([req.forwardedTo], msg):

self.leader_to_coord(req)

else:

print("Forwarded Request to %s" % req.forwardedTo)

def leader_to_coord(self, req):

print("Leader is assuming role of coordinator")

replica_nodes = self.membership_ring.get_replicas_for_key(req.hash)

req.type = req.type[4:]

if req.type == 'get':

msg = messages.getFile(req.hash, req.time_created)

else:

msg = messages.storeFile(req.hash, req.value, req.context, req.time_created)

self.broadcast_message(replica_nodes, msg)

def find_req_for_msg(self, req_ts):

return list(filter(

lambda r: r.time_created == req_ts, self.ongoing_requests

))

# after a \x70, \x80 or \x0B is encountered from a peer, this method is called

def update_request(self, msg, sender, request=None):

print("Updating Request with message ", msg, " from ", sender)

if isinstance(msg, tuple):

if not request:

request = self.find_req_for_msg(msg[-1])

min_num_resp = self.sloppy_R if len(msg) == 3 else self.sloppy_W

else:

request = self.find_req_for_msg(msg.previous_request.time_created)

min_num_resp = 1

if not request:

print("No request found, ", sender, " might have been too slow")

return

elif isinstance(request, list):

request = request[0]

request.responses[sender] = msg

if len(request.responses) >= min_num_resp:

self.complete_request(request)

def coalesce_responses(self, request):

resp_list = list(request.responses.values())

# check if you got a sufficient number of responses

if len(resp_list) < self.sloppy_R:

return None

results = []

for resp in resp_list:

# print(resp)

results.extend([

tup for tup in resp[1] if tup not in results

])

return self.db.sortData(results)

def complete_request(self, request, timer_expired=False):

failed = False

if request.type == 'get':

# if sendbackto is a peer

if request.sendBackTo not in self.client_list:

# this is a response to a for_*

# send the whole request object back to the peer

msg = messages.responseForForward(request)

else:

# compile results from responses and send them to client

# send message to client

msg = messages.getResponse(request.hash, (

self.coalesce_responses(request) if not failed else "Error"

))

elif request.type == 'put':

if len(request.responses) >= self.sloppy_W:

print("Successful put completed for ", request.sendBackTo)

if request.sendBackTo not in self.client_list:

# this is a response to a for_*

# send the whole request object back to the peer

msg = messages.responseForForward(request)

else:

# send success message to client

# check if you were successful

msg = messages.putResponse(request.hash, (

request.value if len(request.responses) >= self.sloppy_W and not failed else "Error"

), request.context)

# if len(request.responses) >= self.sloppy_W and timer_expired:

if timer_expired and len(request.responses) < self.sloppy_Qsize:

target_node = self.membership_ring.get_node_for_key(request.hash)

replica_nodes = self.membership_ring.get_replicas_for_key(request.hash)

all_nodes = set([target_node] + replica_nodes)

missing_reps = set([self.membership_ring.hostname_to_ip[r] for r in all_nodes]) - set(request.responses.keys())

handoff_store_msg = messages.storeFile(request.hash, request.value, request.context, request.time_created)

handoff_msg = messages.handoff(

handoff_store_msg,

missing_reps

)

hons = [

self.membership_ring.get_handoff_node(r)

for r in missing_reps

]

print("Handing off messages for %s to %s" % (", ".join(missing_reps), ", ".join(hons)))

if self.hostname in hons:

self.handle_handoff((handoff_store_msg, missing_reps), self.hostname)

hons.remove(self.hostname)

self.broadcast_message(hons, handoff_msg)

else: # request.type == for_*

# unpack the forwarded request object

data = list(request.responses.values())

if not data:

# del self.req_message_timers[request.time_created]

# request.time_created=time.time()

self.leader_to_coord(request)

T = Timer(self.request_timelimit,

self.complete_request, args=[request], kwargs={"timer_expired": True}

)

T.start()

self.req_message_timers[request.time_created] = T

return

else:

data = data[0]

# if sendbackto is a peer

if request.sendBackTo not in self.client_list:

# unpickle the returned put request

data.previous_request = data.previous_request.previous_request

# send the response object you got back to the peer

# from request.responses (it is the put or get they need)

# if you need to, make req.prev_req = req.prev_req.prev_req

# so it looks like you did the request yourself

msg = messages.responseForForward(data)

elif request.type == 'for_put':

msg = messages.putResponse(request.hash, (

request.value if data and len(data.responses) >= self.sloppy_W and not failed else "Error"

), request.context)

else: # for_get

msg = messages.getResponse(request.hash, (

self.coalesce_responses(data) if not failed else "Error"

))

# send msg to request.sendBackTo

# if request.sendBackTo not in self.client_list:

if not request.responded:

print("Sending response back to ", request.sendBackTo)

self.broadcast_message([request.sendBackTo], msg)

request.responded = True

if timer_expired:

# remove request from ongoing list

self.ongoing_requests = list(filter(

lambda r: r.time_created != request.time_created, self.ongoing_requests

))

def perform_operation(self, data, sendBackTo):

if len(data) == 2: # this is a getFile msg

print("%s is asking me to get %s" % (sendBackTo, data[0]))

msg = messages.getFileResponse(data[0], self.db.getFile(data[0]), data[1])

else: # this is a storeFile

print("%s is asking me to store %s" % (sendBackTo, data[0]))

self.db.storeFile(data[0], sendBackTo, data[1], data[2])

msg = messages.storeFileResponse(*data)

self.broadcast_message([sendBackTo], msg)

def handle_forwarded_req(self, prev_req, sendBackTo):

target_node = self.membership_ring.get_node_for_key(prev_req.hash)

print("Handling a forwarded request [ %s, %f ]" % (prev_req.type, prev_req.time_created))

if time.time() - prev_req.time_created < self.request_timelimit:

# someone forwarded you a put request

# if you are the leader, check if you can takecare of it, else,

# start a new put request with this request as the previous one

if prev_req.type == 'put' or prev_req.type == 'for_put':

if self.is_leader:

if target_node == self.hostname:

args = (prev_req.hash, prev_req.value, prev_req.context)

self.start_request('put', args, sendBackTo, prev_req=prev_req)

else:

args = (target_node, prev_req.hash,

prev_req.value, prev_req.context

)

self.start_request('for_put', args, sendBackTo, prev_req)

else: # the leader is forwarding you a put

args = (prev_req.hash, prev_req.value, prev_req.context)

self.start_request('put', args, sendBackTo, prev_req=prev_req)

# someone forwarded you a get request, you need to take care of it

# start new get request with this as the previous one

else: # type is get or for_get

self.start_request('get', prev_req.hash, sendBackTo, prev_req)

def _send_data_to_peer(self, target_node, data, sendBackTo):

# create for_put request

self.start_request('for_put', [target_node] + data, sendBackTo=sendBackTo)

def _request_data_from_peer(self, target_node, data, sendBackTo):

self.start_request('for_get', (target_node, data), sendBackTo=sendBackTo)

def _create_socket(self, hostname):

"""Creates a socket to the host and adds it connections dict. Returns created socket object."""

s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)

s.setblocking(False)

s.settimeout(1) # 10 seconds

try:

s.connect((hostname, self.tcp_port))

self.connections[socket.gethostbyname(hostname)] = s

return s

except Exception as e:

if hostname not in self.client_list:

print("Error creating connection to %s: %s" % (hostname, e))

return None

# this is where we need to handle hinted handoff if a

# peer is not responsive by asking another peer to hold the

# message until the correct node recovers

def broadcast_message(self, nodes, msg):

fails = []

for node in nodes:

c = self.connections.get(node, self._create_socket(node))

if not c:

fails.append(node)

continue

c.sendall(msg)