def rcv_clientput(self, msg): preference_list, avoided = DynamoNode.chash.find_nodes( msg.key, DynamoNode.N, self.failed_nodes) avoided = avoided[:DynamoNode.N] non_extra_count = DynamoNode.N - len(avoided) if self not in preference_list: coordinator = preference_list[0] Framework.forward_message(msg, coordinator) else: seqno = self.generate_seq_num() metadata = copy.deepcopy(msg.metadata) metadata.update(self.name, seqno) self.pending_put_rsp = set() self.pending_put_msg = msg reqcount = 0 for i, node in enumerate(preference_list): handoff = avoided if (i >= non_extra_count) else None putmsg = PutReq(self, node, msg.key, msg.value, metadata, msg_id=seqno, handoff=handoff) Framework.send_message(putmsg) reqcount += 1 if reqcount >= DynamoNode.N: break
def rcv_clientput(self, msg): preference_list = DynamoNode.chash.find_nodes(msg.key, DynamoNode.N, self.failed_nodes)[0] # Determine if we are in the list if self not in preference_list: # Forward to the coordinator for this key _logger.info("put(%s=%s) maps to %s", msg.key, msg.value, preference_list) coordinator = preference_list[0] Framework.forward_message(msg, coordinator) else: # Use an incrementing local sequence number to distinguish # multiple requests for the same key seqno = self.generate_sequence_number() _logger.info("%s, %d: put %s=%s", self, seqno, msg.key, msg.value) metadata = (self.name, seqno) # For now, metadata is just sequence number at coordinator # Send out to preference list, and keep track of who has replied self.pending_req[PutReq][seqno] = set() self.pending_put_rsp[seqno] = set() self.pending_put_msg[seqno] = msg reqcount = 0 for node in preference_list: # Send message to get node in preference list to store putmsg = PutReq(self, node, msg.key, msg.value, metadata, msg_id=seqno) self.pending_req[PutReq][seqno].add(putmsg) Framework.send_message(putmsg) reqcount = reqcount + 1 if reqcount >= DynamoNode.N: # preference_list may have more than N entries to allow for failed nodes break
def rcv_clientput(self, msg): preference_list, avoided = DynamoNode.chash.find_nodes(msg.key, DynamoNode.N, self.failed_nodes) non_extra_count = DynamoNode.N - len(avoided) # Determine if we are in the list if self not in preference_list: # Forward to the coordinator for this key _logger.info("put(%s=%s) maps to %s", msg.key, msg.value, preference_list) coordinator = preference_list[0] Framework.forward_message(msg, coordinator) else: # Use an incrementing local sequence number to distinguish # multiple requests for the same key seqno = self.generate_sequence_number() _logger.info("%s, %d: put %s=%s", self, seqno, msg.key, msg.value) metadata = (self.name, seqno) # For now, metadata is just sequence number at coordinator # Send out to preference list, and keep track of who has replied self.pending_req[PutReq][seqno] = set() self.pending_put_rsp[seqno] = set() self.pending_put_msg[seqno] = msg reqcount = 0 for ii, node in enumerate(preference_list): if ii >= non_extra_count: # This is an extra node that's only include because of a failed node handoff = avoided else: handoff = None # Send message to get node in preference list to store putmsg = PutReq(self, node, msg.key, msg.value, metadata, msg_id=seqno, handoff=handoff) self.pending_req[PutReq][seqno].add(putmsg) Framework.send_message(putmsg) reqcount = reqcount + 1 if reqcount >= DynamoNode.N: # preference_list may have more than N entries to allow for failed nodes break
def rcv_clientput(self, msg): preference_list = DynamoNode.chash.find_nodes(msg.key, DynamoNode.N)[0] # Determine if we are in the list if self not in preference_list: # Forward to the coordinator for this key _logger.info("put(%s=%s) maps to %s", msg.key, msg.value, preference_list) coordinator = preference_list[0] Framework.forward_message(msg, coordinator) else: # Use an incrementing local sequence number to distinguish # multiple requests for the same key seqno = self.generate_sequence_number() _logger.info("%s, %d: put %s=%s", self, seqno, msg.key, msg.value) metadata = (self.name, seqno) # For now, metadata is just sequence number at coordinator # Send out to preference list, and keep track of who has replied self.pending_put_rsp[seqno] = set() self.pending_put_msg[seqno] = msg reqcount = 0 for node in preference_list: # Send message to get node in preference list to store putmsg = PutReq(self, node, msg.key, msg.value, metadata, msg_id=seqno) Framework.send_message(putmsg) reqcount = reqcount + 1 if reqcount >= DynamoNode.N: # preference_list may have more than N entries to allow for failed nodes break
def rcv_clientput(self, msg): preference_list, avoided = DynamoNode.chash.find_nodes( msg.key, DynamoNode.N, self.failed_nodes) # Only track avoided nodes that would have been part of the original preference list avoided = avoided[:DynamoNode.N] non_extra_count = DynamoNode.N - len(avoided) # Determine if we are in the list if self not in preference_list: # Forward to the coordinator for this key _logger.info("put(%s=%s) maps to %s", msg.key, msg.value, preference_list) coordinator = preference_list[0] Framework.forward_message(msg, coordinator) else: # Use an incrementing local sequence number to distinguish # multiple requests for the same key seqno = self.generate_sequence_number() _logger.info("%s, %d: put %s=%s", self, seqno, msg.key, msg.value) metadata = ( self.name, seqno ) # For now, metadata is just sequence number at coordinator # Send out to preference list, and keep track of who has replied self.pending_req[PutReq][seqno] = set() self.pending_put_rsp[seqno] = set() self.pending_put_msg[seqno] = msg reqcount = 0 for ii, node in enumerate(preference_list): if ii >= non_extra_count: # This is an extra node that's only include because of a failed node handoff = avoided else: handoff = None # Send message to get node in preference list to store putmsg = PutReq(self, node, msg.key, msg.value, metadata, msg_id=seqno, handoff=handoff) self.pending_req[PutReq][seqno].add(putmsg) Framework.send_message(putmsg) reqcount = reqcount + 1 if reqcount >= DynamoNode.N: # preference_list may have more than N entries to allow for failed nodes break
def rcv_clientget(self, msg): preference_list = DynamoNode.chash.find_nodes(msg.key, DynamoNode.N)[0] # Determine if we are in the list if self not in preference_list: # Forward to the coordinator for this key _logger.info("get(%s=?) maps to %s", msg.key, preference_list) coordinator = preference_list[0] Framework.forward_message(msg, coordinator) else: seqno = self.generate_sequence_number() self.pending_get_rsp[seqno] = set() self.pending_get_msg[seqno] = msg reqcount = 0 for node in preference_list: getmsg = GetReq(self, node, msg.key, msg_id=seqno) Framework.send_message(getmsg) reqcount = reqcount + 1 if reqcount >= DynamoNode.N: # preference_list may have more than N entries to allow for failed nodes break
def rcv_clientget(self, msg): preference_list = DynamoNode.chash.find_nodes(msg.key, DynamoNode.N, self.failed_nodes)[0] # Determine if we are in the list #modified if self.addr not in preference_list: ################################ # Forward to the coordinator for this key _logger.info("get(%s=?) maps to %s", msg.key, preference_list) for e in preference_list: con = self.connections[self.servers.index(e)] result = Framework.forward_message(msg, con, e) if result is not False: break if e not in self.failed_nodes: self.failed_nodes.append(e) return result else: seqno = self.generate_sequence_number() self.pending_req[GetReq][seqno] = set() self.pending_get_rsp[seqno] = set() self.pending_get_msg[seqno] = msg reqcount = 0 #modified value = [] ################ for node in preference_list: getmsg = GetReq(self.addr, node, msg.key, msg_id=seqno) self.pending_req[GetReq][seqno].add(getmsg) #modified con = self.connections[self.servers.index(getmsg.to_node)] result = Framework.send_message(getmsg, con) if result is not False: value.append(result) reqcount = reqcount + 1 ############################################ if reqcount >= DynamoNode.N: # preference_list may have more than N entries to allow for failed nodes break # print value #modified # no value for this key if len(value) < 1: return False ########################## return value
class DynamoNode(Node): timer_priority = 20 T = 10 # Number of repeats for nodes in consistent hash table N = 2 # Number of nodes to replicate at W = 1 # Number of nodes that need to reply to a write operation R = 1 # Number of nodes that need to reply to a read operation nodelist = [] chash = ConsistentHashTable(nodelist, T) def __init__(self, addr, config_file='server_config'): super(DynamoNode, self).__init__() self.framework = Framework() self.m_addr = "127.0.0.1:29009" self.local_store = MerkleTree() # key => (value, metadata) self.pending_put_rsp = {} # seqno => set of nodes that have stored self.pending_put_msg = {} # seqno => original client message self.pending_get_rsp = {} # seqno => set of (node, value, metadata) tuples self.pending_get_msg = {} # seqno => original client message # seqno => set of requests sent to other nodes, for each message class self.pending_req = {PutReq: {}, GetReq: {}} self.failed_nodes = [] self.pending_handoffs = {} self.result = {} self.reduceResult = {} self.MapReduceDB = leveldb.LevelDB('./' + addr + '_mrdb') self.mapDict = {} # Rebuild the consistent hash table self.addr = addr self.servers = [] self.db = leveldb.LevelDB('./' + addr + '_db') c = zerorpc.Client(timeout=3) c.connect('tcp://' + self.m_addr) try: c.add_node(self.addr) c.close() except: pass self.pool = gevent.pool.Group() self.pool.spawn(self.retry_failed_node) self.framework.setDynamo(self) def getNodeList(self, servers): print servers for i, server in enumerate(list(servers)): DynamoNode.nodelist.append(server) DynamoNode.chash = ConsistentHashTable(DynamoNode.nodelist, DynamoNode.T) # PART reset @classmethod def reset(cls): cls.nodelist = [] cls.chash = ConsistentHashTable(cls.nodelist, cls.T) # PART storage def store(self, key, value, metadata): self.db.Put(key,pickle.dumps((value,metadata))) def retrieve(self, key): try: return pickle.loads(self.db.Get(key)) except: return (None, None) def are_you_there(self): return # PART retry_failed_node def retry_failed_node(self): # Permanently repeating timer while True: if self.failed_nodes: for node in self.failed_nodes: c = zerorpc.Client(timeout=1) c.connect('tcp://' + str(node)) try: c.are_you_there() c.close() self.recovery(node) except: continue time.sleep(1) def recovery(self, node): # Remove all instances of recovered node from failed node list recovered_node = node while recovered_node in self.failed_nodes: self.failed_nodes.remove(recovered_node) if recovered_node in self.pending_handoffs: for key in self.pending_handoffs[recovered_node]: # Send our latest value for this key (value, metadata) = self.retrieve(key) putmsg = PutReq(self.addr, recovered_node, key, value, metadata) self.framework.send_message(putmsg) self.framework.schedule() del self.pending_handoffs[recovered_node] def retry_request(self, reqmsg): self.failed_nodes.append(reqmsg.to_node) if not isinstance(reqmsg, DynamoRequestMessage): return # Send the request to an additional node by regenerating the preference list preference_list = DynamoNode.chash.find_nodes(reqmsg.key, DynamoNode.N, self.failed_nodes)[0] kls = reqmsg.__class__ # Check the pending-request list for this type of request message if kls in self.pending_req and reqmsg.msg_id in self.pending_req[kls]: for node in preference_list: try: if node not in [req.to_node for req in self.pending_req[kls][reqmsg.msg_id]]: # Found a node on the new preference list that hasn't been sent the request. # Send it a copy newreqmsg = copy.copy(reqmsg) newreqmsg.to_node = node self.pending_req[kls][reqmsg.msg_id].add(newreqmsg) self.framework.send_message(newreqmsg) self.framework.schedule() except: pass # PART rcv_clientput def rcv_clientput(self, msg): preference_list, avoided = DynamoNode.chash.find_nodes(msg.key, DynamoNode.N, self.failed_nodes) non_extra_count = DynamoNode.N - len(avoided) # Determine if we are in the list if self.addr not in preference_list: # Forward to the coordinator for this key _logger.info("put(%s=%s) maps to %s", msg.key, msg.value, preference_list) coordinator = preference_list[0] self.framework.forward_message(msg, coordinator) self.framework.schedule() else: # Use an incrementing local sequence number to distinguish # multiple requests for the same key seqno = self.generate_sequence_number() _logger.info("%s, %d: put %s=%s", self, seqno, msg.key, msg.value) # The metadata for a key is passed in by the client, and updated by the coordinator node. metadata = copy.deepcopy(msg.metadata) metadata.update(self.name, seqno) # Send out to preference list, and keep track of who has replied self.pending_req[PutReq][seqno] = set() self.pending_put_rsp[seqno] = set() self.pending_put_msg[seqno] = msg reqcount = 0 for ii, node in enumerate(preference_list): if ii >= non_extra_count: # This is an extra node that's only include because of a failed node handoff = avoided else: handoff = None # Send message to get node in preference list to store putmsg = PutReq(self.addr, node, msg.key, msg.value, metadata, msg_id=seqno, handoff=handoff) try: self.pending_req[PutReq][seqno].add(putmsg) except: pass self.framework.send_message(putmsg) self.framework.schedule() reqcount = reqcount + 1 if reqcount >= DynamoNode.N: # preference_list may have more than N entries to allow for failed nodes break # PART rcv_clientget def rcv_clientget(self, msg): preference_list = DynamoNode.chash.find_nodes(msg.key, DynamoNode.N, self.failed_nodes)[0] # Determine if we are in the list #print preference_list if self.addr not in preference_list: # Forward to the coordinator for this key _logger.info("get(%s=?) maps to %s", msg.key, preference_list) coordinator = preference_list[0] self.framework.forward_message(msg, coordinator) self.framework.schedule() else: seqno = self.generate_sequence_number() self.pending_req[GetReq][seqno] = set() self.pending_get_rsp[seqno] = set() self.pending_get_msg[seqno] = msg reqcount = 0 for node in preference_list: getmsg = GetReq(self.addr, node, msg.key, msg_id=seqno) try: self.pending_req[GetReq][seqno].add(getmsg) except: pass self.framework.send_message(getmsg) self.framework.schedule() reqcount = reqcount + 1 if reqcount >= DynamoNode.N: # preference_list may have more than N entries to allow for failed nodes break # PART rcv_put def rcv_put(self, putmsg): _logger.info("%s: store %s=%s", self, putmsg.key, putmsg.value) self.store(putmsg.key, putmsg.value, putmsg.metadata) if putmsg.handoff is not None: for failed_node in putmsg.handoff: if failed_node not in self.failed_nodes: self.failed_nodes.append(failed_node) if failed_node not in self.pending_handoffs: self.pending_handoffs[failed_node] = set() self.pending_handoffs[failed_node].add(putmsg.key) putrsp = PutRsp(putmsg) self.framework.send_message(putrsp) self.framework.schedule() # PART rcv_putrsp def rcv_putrsp(self, putrsp): seqno = putrsp.msg_id if seqno in self.pending_put_rsp: self.pending_put_rsp[seqno].add(putrsp.from_node) if len(self.pending_put_rsp[seqno]) >= DynamoNode.W: _logger.info("%s: written %d copies of %s=%s so done", self, DynamoNode.W, putrsp.key, putrsp.value) #_logger.debug(" copies at %s", [node.name for node in self.pending_put_rsp[seqno]]) # Tidy up tracking data structures original_msg = self.pending_put_msg[seqno] del self.pending_req[PutReq][seqno] del self.pending_put_rsp[seqno] del self.pending_put_msg[seqno] # Reply to the original client client_putrsp = ClientPutRsp(original_msg, putrsp.metadata) self.framework.send_message(client_putrsp) self.framework.schedule() else: pass # Superfluous reply # PART rcv_get def rcv_get(self, getmsg): _logger.info("%s: retrieve %s=?", self, getmsg.key) (value, metadata) = self.retrieve(getmsg.key) getrsp = GetRsp(getmsg, value, metadata) self.framework.send_message(getrsp) self.framework.schedule() # PART rcv_getrsp def rcv_getrsp(self, getrsp): seqno = getrsp.msg_id if seqno in self.pending_get_rsp: self.pending_get_rsp[seqno].add((getrsp.from_node, getrsp.value, getrsp.metadata)) #print len(self.pending_get_rsp[seqno]) if len(self.pending_get_rsp[seqno]) >= DynamoNode.R: _logger.info("%s: read %d copies of %s=? so done", self, DynamoNode.R, getrsp.key) #_logger.debug(" copies at %s", [(node.name, value) for (node, value, _) in self.pending_get_rsp[seqno]]) # Coalesce all compatible (value, metadata) pairs across the responses results = VectorClock.coalesce2([(value, metadata) for (node, value, metadata) in self.pending_get_rsp[seqno]]) # Tidy up tracking data structures original_msg = self.pending_get_msg[seqno] del self.pending_req[GetReq][seqno] del self.pending_get_rsp[seqno] del self.pending_get_msg[seqno] # Reply to the original client, including all received values client_getrsp = ClientGetRsp(original_msg, [value for (value, metadata) in results], [metadata for (value, metadata) in results]) self.framework.send_message(client_getrsp) #Framework.schedule(timers_to_process=0) self.framework.schedule() else: pass # Superfluous reply # PART rcvmsg def rcvmsg(self, msg): msg = pickle.loads(msg) if isinstance(msg, ClientPut): #print 'get ClientPut' self.rcv_clientput(msg) elif isinstance(msg, PutReq): #print 'get PutReq' self.rcv_put(msg) elif isinstance(msg, PutRsp): #print 'get PutRsp' self.rcv_putrsp(msg) elif isinstance(msg, ClientGet): #print 'get ClientGet' self.rcv_clientget(msg) elif isinstance(msg, GetReq): #print 'get GetReq' self.rcv_get(msg) elif isinstance(msg, GetRsp): #print 'get GetRsp' self.rcv_getrsp(msg) elif isinstance(msg, ClientGetRsp): print msg.value self.mapDict[msg.key] = msg.value[0] else: raise TypeError("Unexpected message type %s", msg.__class__) # PART get_contents def get_contents(self): results = [] for key, value in self.local_store.items(): results.append("%s:%s" % (key, value[0])) return results def put_message(self, fromnode, key, value, metadata): #print 'client put!!!' metadata = pickle.loads(metadata) if metadata is None: metadata = VectorClock() else: # A Put operation always implies convergence metadata = VectorClock.converge(metadata) putmsg = ClientPut(fromnode, self.addr, key, value, metadata) self.rcv_clientput(putmsg) #self.framework.send_message(putmsg) # Framework.schedule(timers_to_process=0) #self.framework.schedule() def get_message(self, fromnode, key): #print 'client get!!!' getmsg = ClientGet(fromnode, self.addr, key) #print '++++++++++' + str(key) + '+++++++++++++++++' self.rcv_clientget(getmsg) #self.framework.send_message(getmsg) #self.framework.schedule() #Framework.schedule(timers_to_process=0) def get_keys(self): keys = [] for (key, value) in self.db.RangeIter(): keys.append(key) return keys def startMap(self, keys): self.mapDict.clear() self.result.clear() print 'start mapping' for (k,v) in self.MapReduceDB.RangeIter(): self.MapReduceDB.Delete(k) #print keys for key in keys: self.get_message(self.addr, key) #print 'get key finish' while True: if len(self.mapDict.keys()) == len(keys): break # gevent.sleep(0.5) #print self.mapDict #self.count(self.mapDict) for key in self.mapDict.keys(): self.map(key, self.mapDict[key]) for k in self.result: self.MapReduceDB.Put(k,pickle.dumps(self.result[k])) return def startReduce(self, reduce_nodelist, addr, node): print 'start reducing' #print addr #print dict d = {} print node print self.addr if addr != self.addr: c = zerorpc.Client(timeout = 10) c.connect('tcp://' + addr) try: d = c.send_to_reduce(reduce_nodelist, node) #print 'return' #print d except: pass c.close() else: d = self.send_to_reduce(reduce_nodelist, node) #print 'return' #print d self.reduceResult.clear() for k in d: #print k #print len(pickle.loads(d[k])) self.reduce(k, pickle.loads(d[k])) return self.reduceResult def reduce(self, word, iterator): self.reduceResult[word] = len(iterator) def emit(self, word, sum): if word not in self.result.keys(): self.result[word] = [sum] else: tmplist = [] tmplist = self.result[word] tmplist.append(sum) self.result[word] = tmplist def map(self, name, document): result = {} #for k in d: content = document.__str__().lower() #print 'content is: ' + content words = content.split() for word in words: self.emit(word, 1) def send_to_reduce(self, reduce_nodelist, remoteaddr): print 'sent------------------------' chash = ConsistentHashTable(reduce_nodelist, DynamoNode.T) d = {} print 'remote: ' + remoteaddr #l = [] for (k,v) in self.MapReduceDB.RangeIter(): print chash.find_nodes(k)[0][0] if remoteaddr == chash.find_nodes(k)[0][0]: print k print pickle.loads(self.MapReduceDB.Get(k)) d[k] = self.MapReduceDB.Get(k) #l.append(k) #for k in l: #self.MapReduceDB.Delete(k) return d
def rcv_clientput(self, msg): preference_list, avoided = DynamoNode.chash.find_nodes(msg.key, DynamoNode.N, self.failed_nodes) non_extra_count = DynamoNode.N - len(avoided) # Determine if we are in the list #modified if self.addr not in preference_list: # Forward to the coordinator for this key _logger.info("put(%s=%s) maps to %s", msg.key, msg.value, preference_list) result = True for e in preference_list: con = self.connections[self.servers.index(e)] result = Framework.forward_message(msg, con, e) if result is not False: break if e not in self.failed_nodes: self.failed_nodes.append(e) return result ##################################### else: # Use an incrementing local sequence number to distinguish # multiple requests for the same key seqno = self.generate_sequence_number() _logger.info("%s, %d: put %s=%s", self, seqno, msg.key, msg.value) # The metadata for a key is passed in by the client, and updated by the coordinator node. metadata = copy.deepcopy(msg.metadata) metadata.update(self.name, seqno) # Send out to preference list, and keep track of who has replied self.pending_req[PutReq][seqno] = set() self.pending_put_rsp[seqno] = set() self.pending_put_msg[seqno] = msg reqcount = 0 #modified nodes = [] ##################### for ii, node in enumerate(preference_list): if ii >= non_extra_count: # This is an extra node that's only include because of a failed node handoff = avoided else: handoff = None # Send message to get node in preference list to store putmsg = PutReq(self.addr, node, msg.key, msg.value, metadata, msg_id=seqno, handoff=handoff) self.pending_req[PutReq][seqno].add(putmsg) #modified con = self.connections[self.servers.index(putmsg.to_node)] result = Framework.send_message(putmsg, con) if result is False and putmsg.to_node not in self.failed_nodes: self.failed_nodes.append(putmsg.to_node) if result is not False: nodes.append(node) reqcount = reqcount + 1 #################################### # print "---------------------------------", type(putmsg) if reqcount >= DynamoNode.N: # preference_list may have more than N entries to allow for failed nodes break #modified if reqcount >= DynamoNode.N: return nodes return False