class Node(object):
def __init__(self, is_leader, leader_hostname, my_hostname, tcp_port=13337, sloppy_Qsize=5, sloppy_R=3, sloppy_W=3):
self.ongoing_requests = []
self.is_leader = is_leader
self.leader_hostname = leader_hostname
self.hostname = my_hostname
self.tcp_port = tcp_port
self.my_address = (self.hostname, self.tcp_port)
self.membership_ring = Ring(replica_count=sloppy_Qsize - 1) # Other nodes in the membership
if self.is_leader:
self.membership_ring.add_node(leader_hostname)
# todo: look into this, do we need both?
self.bootstrapping = True
self.is_member = False
# Flag to keep track of a add-node is underway
self._membership_in_progress = False
self.sloppy_Qsize = sloppy_Qsize # total members to replicate on
# number of peers required for a read or write to succeed.
self.sloppy_R = sloppy_R
self.sloppy_W = sloppy_W
# Book keeping for membership messages
self._req_responses = defaultdict(set)
self._sent_req_messages = {}
self._received_req_messages = {}
self._req_sender = {} # Keeps track to sender for add and delete requests
self.current_view = 0 # increment this on every leader election
self.membership_request_id = 0 # increment this on every request sent to peers
# Maintains handoff messages to be sent
# IP : set(handoff messages)
self._handoff_messages = defaultdict(set)
self.handoff_timer = None
self.create_handoff_timer = lambda: Timer(5, self.try_sending_handoffs)
self.log_prefix = os.getcwd()
self.ring_log_file = os.path.join(self.log_prefix, self.hostname + '.ring')
self.db_path = os.path.join(self.log_prefix, self.hostname + '.db')
self.handoff_log = os.path.join(self.log_prefix, self.hostname + '.pickle')
try:
with open(self.ring_log_file, 'r') as f:
hosts = f.readlines()
for h in hosts:
self.membership_ring.add_node(h.strip())
print("Restored membership information from %s" % self.ring_log_file)
except FileNotFoundError:
pass
try:
with open(self.handoff_log, 'rb') as f:
self._handoff_messages = pickle.loads(f.read())
if len(self._handoff_messages) > 0:
self.handoff_timer = self.create_handoff_timer()
self.handoff_timer.start()
print("Restored hand off messages from %s" % self.handoff_log)
except FileNotFoundError:
pass
self.request_timelimit = 2.0
self.req_message_timers = {}
self.db = Storage(self.db_path) # set up sqlite table
# create tcp socket for communication with peers and clients
self.tcp_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.tcp_socket.setblocking(False) # Non-blocking socket
self.tcp_socket.bind((self.hostname, self.tcp_port))
self.tcp_socket.listen(10)
# has hostnames mapped to open sockets
self.connections = {}
self.client_list = set()
def accept_connections(self):
incoming_connections = {self.tcp_socket}
print("Accepting connections...")
while True:
readable, _, _ = select.select(incoming_connections, [], [], 0)
for s in readable:
if s is self.tcp_socket:
connection, client_address = s.accept()
connection.setblocking(False)
incoming_connections.add(connection)
self.connections[client_address[0]] = connection
else:
try:
header = s.recv(5)
except:
print("Connection reset.")
continue
if not header: # remove for connection pool and close socket
incoming_connections.remove(s)
# del self.connections[s.getpeername()[0]]
s.close()
else:
message_len = struct.unpack('!i', header[1:5])[0]
data = b''
while len(data) < message_len:
try:
data += s.recv(message_len - len(data))
except socket.error as err:
pass
self._process_message(header + data, s.getpeername()[0]) # addr is a tuple of hostname and port
def _process_message(self, data, sender):
message_type, data_tuple = messages._unpack_message(data)
message_type_mapping = {
b'\x00': self._process_command,
b'\x01': self._process_req_message,
b'\x10': self._membership_change_message,
b'\xff': self._process_ok_message,
b'\x07': self.perform_operation,
b'\x08': self.perform_operation,
b'\x70': self.update_request,
b'\x80': self.update_request,
b'\x0B': self.update_request,
b'\x0A': self.handle_forwarded_req,
b'\x0C': self.handle_handoff
}
message_type_mapping[message_type](data_tuple, sender)
return
def _process_command(self, user_input, sendBackTo):
"""Process commands"""
self.client_list.add(sendBackTo)
# Maps command to the corresponding function.
# Command arguments are passed as the first argument to the function.
command_registry = { # Possible commands:
"add-node": self.add_node, # 1. add node to membership
"remove-node": self.remove_node, # 2. remove node from membership
"put": self.put_data, # 3. put data
"get": self.get_data, # 4. get data
}
if not user_input:
self._send_req_response_to_client(sendBackTo, "User input empty")
# First word is command. Rest are then arguments.
command, *data = user_input.split(" ")
if command not in command_registry:
self._send_req_response_to_client(sendBackTo, "Invalid command")
# Call the function associated with the command in command_registry
return command_registry[command](data, sendBackTo)
def handle_handoff(self, data, sendBackTo):
# data should have (message, list of hosts to hand data off to)
for h in data[1]:
print("Storing hand off message for %s" % h)
self._handoff_messages[h].add(data[0])
# Save handoff messages to disk
self.sync_handoffs_to_disk()
# check and start timer
if not self.handoff_timer:
self.handoff_timer = self.create_handoff_timer()
self.handoff_timer.start()
def try_sending_handoffs(self):
print("Attempting to send hand off messages...")
updated_handoff_messages = defaultdict(set)
for (host, msgs) in self._handoff_messages.items():
for msg in msgs:
fail = self.broadcast_message([host], msg)
if fail:
updated_handoff_messages[host].add(msg)
self._handoff_messages = updated_handoff_messages
self.sync_handoffs_to_disk()
if len(self._handoff_messages) > 0:
print("Undelivered hand offs. Restarting timer")
self.handoff_timer = self.create_handoff_timer()
self.handoff_timer.start()
return
print("Sent all hand off messages.")
self.handoff_timer = None
def sync_handoffs_to_disk(self):
# Save hand off messages to disk
with open(self.handoff_log, 'wb') as f:
f.write(pickle.dumps(self._handoff_messages))
def add_node(self, data, sender):
"""Add node to membership. data[0] must be the hostname. Initiates 2PC."""
if not self.is_leader:
self._send_req_response_to_client(sender, "Error: This is not the leader")
return
if not data:
self._send_req_response_to_client(sender, "Error: hostname required")
return
if self._membership_in_progress:
self._send_req_response_to_client(sender, "Membership operation in progress. Try again")
return
if data[0] in self.membership_ring:
self._send_req_response_to_client(sender, "Already in the membership")
return
self._membership_in_progress = True
print("Starting add-node operation for %s" % data[0])
new_peer_message = messages.reqMessage(self.current_view, self.membership_request_id, 1, data[0])
# associate hostname to (view_id, req_id)
self._sent_req_messages[(self.current_view, self.membership_request_id)] = (data[0], 1)
self._req_sender[(self.current_view, self.membership_request_id)] = sender
# broadcast to all but leader.
nodes_to_broadcast = self.membership_ring.get_all_hosts()
nodes_to_broadcast.remove(self.hostname)
nodes_to_broadcast.add(data[0]) # Add new host to broadcast list
self.broadcast_message(nodes_to_broadcast, new_peer_message)
t = Timer(self.request_timelimit, self._req_timeout, args=[(self.current_view, self.membership_request_id)])
self.req_message_timers[(self.current_view, self.membership_request_id)] = t
t.start()
self.membership_request_id += 1
# Send a remove node message to everyone and if you are that node, shutdown
def remove_node(self, data, sender):
if not self.is_leader:
self._send_req_response_to_client(sender, "Error: This is not the leader")
return
if not data:
self._send_req_response_to_client(sender, "Error: hostname required")
return
if self._membership_in_progress:
self._send_req_response_to_client(sender, "Membership operation in progress. Try again")
return
if data[0] not in self.membership_ring:
self._send_req_response_to_client(sender, "Cannot remove. Node not in membership")
return
self._membership_in_progress = True
print("Starting remove-node operation for %s" % data[0])
new_peer_message = messages.reqMessage(self.current_view, self.membership_request_id, 2, data[0])
# associate hostname to (view_id, req_id)
self._sent_req_messages[(self.current_view, self.membership_request_id)] = (data[0], 2)
self._req_sender[(self.current_view, self.membership_request_id)] = sender
# broadcast to all but leader.
nodes_to_broadcast = self.membership_ring.get_all_hosts()
nodes_to_broadcast.remove(self.hostname)
nodes_to_broadcast.add(data[0]) # Add new host to broadcast list
self.broadcast_message(nodes_to_broadcast, new_peer_message)
t = Timer(self.request_timelimit, self._req_timeout, args=[(self.current_view, self.membership_request_id)])
self.req_message_timers[(self.current_view, self.membership_request_id)] = t
t.start()
self.membership_request_id += 1
# self.membership_ring.remove_node(data[0])
def put_data(self, data, sendBackTo):
if len(data) != 3:
return "Error: Invalid operands\nInput: (<key>,<prev version>,<value>)"
data = [data[0], json.loads(data[1]), data[2]]
key = data[0]
prev = data[1]
value = data[2]
target_node = self.membership_ring.get_node_for_key(data[0])
if not self.is_leader:
# forward request to leader for client
return self._send_data_to_peer(self.leader_hostname, data, sendBackTo)
else: # I am the leader
if target_node == self.hostname:
# I'm processing a request for a client directly
self.start_request('put', data, sendBackTo=sendBackTo)
return
else: # I am forwarding a request from the client to the correct node
return self._send_data_to_peer(target_node, data, sendBackTo)
def get_data(self, data, sendBackTo):
"""Retrieve V for given K from the database. data[0] must be the key"""
if not data:
return "Error: key required"
target_node = self.membership_ring.get_node_for_key(data[0])
# if I can do it myself
if target_node == self.hostname:
# I am processing a request for a client directly
self.start_request('get', data[0], sendBackTo=sendBackTo)
else: # forward the client request to the peer incharge of req
self._request_data_from_peer(target_node, data[0], sendBackTo)
def _process_req_message(self, data, sender):
# data = (view_id, req_id, operation, address)
(view_id, req_id, operation, address) = data
print("Processed request message to add %s" % address)
# save the message type
self._received_req_messages[(view_id, req_id)] = (address, operation)
ok_message = messages.okMessage(view_id, req_id)
self.connections.get(sender, self._create_socket(sender)).sendall(ok_message)
def _process_ok_message(self, data, sender):
self._req_responses[data].add(sender)
(new_peer_hostname, operation) = self._sent_req_messages[data]
required_responses = len(self.membership_ring) if operation == 1 else (len(self.membership_ring) - 1)
# number of replies equal number of *followers* already in the ring, add peer to membership
if len(self._req_responses[data]) == required_responses:
# Cancel timer
t = self.req_message_timers.get(data, None)
if not t:
return
t.cancel()
# Send newViewMessage
# self.current_view += 1
if operation == 1: # adding new node
self.membership_ring.add_node(new_peer_hostname)
hosts_to_send = self.membership_ring.get_all_hosts()
nodes_to_broadcast = self.membership_ring.get_all_hosts()
else:
hosts_to_send = (new_peer_hostname, )
nodes_to_broadcast = self.membership_ring.get_all_hosts()
nodes_to_broadcast.remove(new_peer_hostname)
self.membership_ring.remove_node(new_peer_hostname)
membership_change_msg = messages.membershipChange(self.current_view, operation, hosts_to_send)
nodes_to_broadcast.remove(self.hostname)
self.broadcast_message(nodes_to_broadcast, membership_change_msg)
print("Successfully %s %s." % ("added" if operation == 1 else "removed", new_peer_hostname))
client = self._req_sender.get(data, None)
self._send_req_response_to_client(client, "Successfully %s %s." %
("added" if operation == 1 else "removed", new_peer_hostname))
self._membership_in_progress = False # reset state to accept new connections
def _membership_change_message(self, data, sender):
(view_id, operation, peers) = data
# self.current_view = view_id
if operation == 1: # add nodes
for p in peers:
if p not in self.membership_ring:
self.membership_ring.add_node(p)
if operation == 2:
for p in peers:
if p in self.membership_ring:
self.membership_ring.remove_node(p)
with open(self.ring_log_file, 'w') as f:
for node in self.membership_ring.get_all_hosts():
f.write(node + '\n')
print("Successfully modified membership ring. Total members: %d" % len(self.membership_ring.get_all_hosts()))
print("Current members: %s" % ", ".join(self.membership_ring.get_all_hosts()))
print("Keys to manage: ", self.membership_ring.get_key_range(self.hostname))
def _req_timeout(self, req_id):
print("Error adding node to network. One or more nodes is offline.")
# failed_hosts = set(self.membership_ring.get_all_hosts()) - self._req_responses[req_id]
# failed_hosts.remove()
sender = self._req_sender.get(req_id, None)
self._send_req_response_to_client(sender, "Failed to add node to the network")
self._membership_in_progress = False
def _send_req_response_to_client(self, client, message):
msg = messages.responseForForward(message)
self.broadcast_message([client], msg)
# request format:
# object which contains
# type
# sendBackTo
# forwardedTo =None if type is not for_*
# hash
# value =None if type is get or forget
# context =None if type is get or forget
# responses = { sender:msg, sender2:msg2... }
# args format is determined by type:
# type='get', args='hash'
# type='put', args=('hash','value',{context})
# type='for_get', args=(target_node,'hash')
# type='for_put', args=(target_node, 'hash','value',{context})
# Type can be 'put', 'get', 'for_put', 'for_get'
# 'for_*' if for requests that must be handled by a different peer
# then when the response is returned, complete_request will send the
# output to the correct client or peer (or stdin)
def start_request(self, rtype, args, sendBackTo, prev_req=None):
print("%s request from %s: %s" % (rtype, sendBackTo, args))
req = Request(rtype, args, sendBackTo, previous_request=prev_req) # create request obj
self.ongoing_requests.append(req) # set as ongoing
target_node = self.membership_ring.get_node_for_key(req.hash)
replica_nodes = self.membership_ring.get_replicas_for_key(req.hash)
T = Timer(self.request_timelimit + (1 if rtype[:3] == 'for' else 0),
self.complete_request, args=[req], kwargs={"timer_expired": True}
)
T.start()
self.req_message_timers[req.time_created] = T
# Find out if you can respond to this request
if rtype == 'get':
# add my information to the request
result = self.db.getFile(args)
my_resp = messages.getFileResponse(args, result, req.time_created)
self.update_request(messages._unpack_message(my_resp)[1], socket.gethostbyname(self.hostname), req)
# send the getFile message to everyone in the replication range
msg = messages.getFile(req.hash, req.time_created)
# this function will need to handle hinted handoff
print("Sending getFile message to %s" % ", ".join(replica_nodes))
fails = self.broadcast_message(replica_nodes, msg)
if fails:
print("Failed to send get msg to %s" % ', '.join(fails))
elif rtype == 'put':
self.db.storeFile(args[0], socket.gethostbyname(self.hostname), args[1], args[2])
my_resp = messages.storeFileResponse(args[0], args[1], args[2], req.time_created)
# add my information to the request
self.update_request(messages._unpack_message(my_resp)[1], socket.gethostbyname(self.hostname), req)
# send the storeFile message to everyone in the replication range
msg = messages.storeFile(req.hash, req.value, req.context, req.time_created)
# this function will need to handle hinted handoff
print("Sending storeFile message to %s" % ", ".join(replica_nodes))
fails = self.broadcast_message(replica_nodes, msg)
if fails:
print("Failed to send put msg to %s" % ', '.join(fails))
else:
msg = messages.forwardedReq(req)
# forward message to target node
# self.connections[req.forwardedTo].sendall(msg)
if self.broadcast_message([req.forwardedTo], msg):
self.leader_to_coord(req)
else:
print("Forwarded Request to %s" % req.forwardedTo)
def leader_to_coord(self, req):
print("Leader is assuming role of coordinator")
replica_nodes = self.membership_ring.get_replicas_for_key(req.hash)
req.type = req.type[4:]
if req.type == 'get':
msg = messages.getFile(req.hash, req.time_created)
else:
msg = messages.storeFile(req.hash, req.value, req.context, req.time_created)
self.broadcast_message(replica_nodes, msg)
def find_req_for_msg(self, req_ts):
return list(filter(
lambda r: r.time_created == req_ts, self.ongoing_requests
))
# after a \x70, \x80 or \x0B is encountered from a peer, this method is called
def update_request(self, msg, sender, request=None):
print("Updating Request with message ", msg, " from ", sender)
if isinstance(msg, tuple):
if not request:
request = self.find_req_for_msg(msg[-1])
min_num_resp = self.sloppy_R if len(msg) == 3 else self.sloppy_W
else:
request = self.find_req_for_msg(msg.previous_request.time_created)
min_num_resp = 1
if not request:
print("No request found, ", sender, " might have been too slow")
return
elif isinstance(request, list):
request = request[0]
request.responses[sender] = msg
if len(request.responses) >= min_num_resp:
self.complete_request(request)
def coalesce_responses(self, request):
resp_list = list(request.responses.values())
# check if you got a sufficient number of responses
if len(resp_list) < self.sloppy_R:
return None
results = []
for resp in resp_list:
# print(resp)
results.extend([
tup for tup in resp[1] if tup not in results
])
return self.db.sortData(results)
def complete_request(self, request, timer_expired=False):
failed = False
if request.type == 'get':
# if sendbackto is a peer
if request.sendBackTo not in self.client_list:
# this is a response to a for_*
# send the whole request object back to the peer
msg = messages.responseForForward(request)
else:
# compile results from responses and send them to client
# send message to client
msg = messages.getResponse(request.hash, (
self.coalesce_responses(request) if not failed else "Error"
))
elif request.type == 'put':
if len(request.responses) >= self.sloppy_W:
print("Successful put completed for ", request.sendBackTo)
if request.sendBackTo not in self.client_list:
# this is a response to a for_*
# send the whole request object back to the peer
msg = messages.responseForForward(request)
else:
# send success message to client
# check if you were successful
msg = messages.putResponse(request.hash, (
request.value if len(request.responses) >= self.sloppy_W and not failed else "Error"
), request.context)
# if len(request.responses) >= self.sloppy_W and timer_expired:
if timer_expired and len(request.responses) < self.sloppy_Qsize:
target_node = self.membership_ring.get_node_for_key(request.hash)
replica_nodes = self.membership_ring.get_replicas_for_key(request.hash)
all_nodes = set([target_node] + replica_nodes)
missing_reps = set([self.membership_ring.hostname_to_ip[r] for r in all_nodes]) - set(request.responses.keys())
handoff_store_msg = messages.storeFile(request.hash, request.value, request.context, request.time_created)
handoff_msg = messages.handoff(
handoff_store_msg,
missing_reps
)
hons = [
self.membership_ring.get_handoff_node(r)
for r in missing_reps
]
print("Handing off messages for %s to %s" % (", ".join(missing_reps), ", ".join(hons)))
if self.hostname in hons:
self.handle_handoff((handoff_store_msg, missing_reps), self.hostname)
hons.remove(self.hostname)
self.broadcast_message(hons, handoff_msg)
else: # request.type == for_*
# unpack the forwarded request object
data = list(request.responses.values())
if not data:
# del self.req_message_timers[request.time_created]
# request.time_created=time.time()
self.leader_to_coord(request)
T = Timer(self.request_timelimit,
self.complete_request, args=[request], kwargs={"timer_expired": True}
)
T.start()
self.req_message_timers[request.time_created] = T
return
else:
data = data[0]
# if sendbackto is a peer
if request.sendBackTo not in self.client_list:
# unpickle the returned put request
data.previous_request = data.previous_request.previous_request
# send the response object you got back to the peer
# from request.responses (it is the put or get they need)
# if you need to, make req.prev_req = req.prev_req.prev_req
# so it looks like you did the request yourself
msg = messages.responseForForward(data)
elif request.type == 'for_put':
msg = messages.putResponse(request.hash, (
request.value if data and len(data.responses) >= self.sloppy_W and not failed else "Error"
), request.context)
else: # for_get
msg = messages.getResponse(request.hash, (
self.coalesce_responses(data) if not failed else "Error"
))
# send msg to request.sendBackTo
# if request.sendBackTo not in self.client_list:
if not request.responded:
print("Sending response back to ", request.sendBackTo)
self.broadcast_message([request.sendBackTo], msg)
request.responded = True
if timer_expired:
# remove request from ongoing list
self.ongoing_requests = list(filter(
lambda r: r.time_created != request.time_created, self.ongoing_requests
))
def perform_operation(self, data, sendBackTo):
if len(data) == 2: # this is a getFile msg
print("%s is asking me to get %s" % (sendBackTo, data[0]))
msg = messages.getFileResponse(data[0], self.db.getFile(data[0]), data[1])
else: # this is a storeFile
print("%s is asking me to store %s" % (sendBackTo, data[0]))
self.db.storeFile(data[0], sendBackTo, data[1], data[2])
msg = messages.storeFileResponse(*data)
self.broadcast_message([sendBackTo], msg)
def handle_forwarded_req(self, prev_req, sendBackTo):
target_node = self.membership_ring.get_node_for_key(prev_req.hash)
print("Handling a forwarded request [ %s, %f ]" % (prev_req.type, prev_req.time_created))
if time.time() - prev_req.time_created < self.request_timelimit:
# someone forwarded you a put request
# if you are the leader, check if you can takecare of it, else,
# start a new put request with this request as the previous one
if prev_req.type == 'put' or prev_req.type == 'for_put':
if self.is_leader:
if target_node == self.hostname:
args = (prev_req.hash, prev_req.value, prev_req.context)
self.start_request('put', args, sendBackTo, prev_req=prev_req)
else:
args = (target_node, prev_req.hash,
prev_req.value, prev_req.context
)
self.start_request('for_put', args, sendBackTo, prev_req)
else: # the leader is forwarding you a put
args = (prev_req.hash, prev_req.value, prev_req.context)
self.start_request('put', args, sendBackTo, prev_req=prev_req)
# someone forwarded you a get request, you need to take care of it
# start new get request with this as the previous one
else: # type is get or for_get
self.start_request('get', prev_req.hash, sendBackTo, prev_req)
def _send_data_to_peer(self, target_node, data, sendBackTo):
# create for_put request
self.start_request('for_put', [target_node] + data, sendBackTo=sendBackTo)
def _request_data_from_peer(self, target_node, data, sendBackTo):
self.start_request('for_get', (target_node, data), sendBackTo=sendBackTo)
def _create_socket(self, hostname):
"""Creates a socket to the host and adds it connections dict. Returns created socket object."""
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.setblocking(False)
s.settimeout(1) # 10 seconds
try:
s.connect((hostname, self.tcp_port))
self.connections[socket.gethostbyname(hostname)] = s
return s
except Exception as e:
if hostname not in self.client_list:
print("Error creating connection to %s: %s" % (hostname, e))
return None
# this is where we need to handle hinted handoff if a
# peer is not responsive by asking another peer to hold the
# message until the correct node recovers
def broadcast_message(self, nodes, msg):
fails = []
for node in nodes:
c = self.connections.get(node, self._create_socket(node))
if not c:
fails.append(node)
continue
c.sendall(msg)
return fails
