class DFS(object): def __init__(self, local_address, remote_address = None): self.local_ = Local(local_address, remote_address) def read_wrap(msg): return self._read(msg) def write_wrap(msg): return self._write(msg) def attr_wrap(msg): return self._attr(msg) self.data_ = {} self.attr_ = {} self.shutdown_ = False self.local_.register_command("read", read_wrap) self.local_.register_command("write", write_wrap) self.local_.register_command("attr", attr_wrap) self.local_.start()
class Chat(object): """Data structure that represents a member of a chat network.""" def __init__(self, local_address, remote_address, name): """ :param local_address: address of the local node. :param remote_address: address of the remote node to bootstrap from. :param name: name belonging to the local node (name of the local user) """ self.local_ = Local(local_address, remote_address) # local chord node self.local_.register_command("msg", self._accept_msg) self.local_.register_command("named_msg", self._accept_named_msg) self.local_.register_command("join", self._accept_join) self.local_.register_command("leave", self._accept_leave) self.local_.register_command("channel_msg", self._accept_channel_msg) self.local_.register_command("migrate", self._accept_migrate) self.local_.register_command("name", self._accept_name) self.local_.register_command("delete_name", self._accept_delete_name) self.local_.set_notify_handler(self._predecessor_change) self.local_.start() # starting the local chord node self.msg_handler_ = None # handler for outputting chat messages self.mntnd_chnls = dict() self.mntnd_names = dict() self.joined_channels = set() self.name_ = name name_hash = hash_string(name) print('registering our name') node = self.local_.find_successor(name_hash) node.user_command("name", json.dumps({ 'name': name, 'id': self.local_.id() })) self.local_.stabilize def set_msg_handler(self, handler): """Sets handler of the incoming messages. The handler must be a function accepting a message as its first parameter.""" self.msg_handler_ = handler def _accept_name(self, msg): """Handler of a name command.""" try: msg = json.loads(msg) self.mntnd_names[msg['name']] = msg['id'] print('we map name ' + msg['name'] + ' to id ' + str(msg['id'])) except (KeyError, ValueError) as error: print("bad message accepted", error) return '' def _accept_delete_name(self, msg): """Handler of a name command.""" try: msg = json.loads(msg) name = msg['name'] if name not in self.mntnd_names: print("accepted delete command for unknown name " + name) return '' del self.mntnd_names[name] print('deleted name ' + name + ' from maintained names') except (KeyError, ValueError) as error: print("bad message accepted", error) return '' def _accept_named_msg(self, msg): """Handler of a msg command.""" try: msg = json.loads(msg) name = msg['name'] if name == self.name_: if self.msg_handler_ is not None: self.msg_handler_(msg['msg']) return '' if not name in self.mntnd_names: print('received a message for unknown name ' + name) return '' recp_id = self.mntnd_names[name] print('received a message for name ' + name + ', sending it to ' + str(recp_id)) node = self.local_.find_successor(recp_id) node.user_command( "msg", json.dumps({ 'key': str(recp_id), 'value': msg['msg'] })) except (KeyError, ValueError) as error: print("bad message accepted", error) return '' def send_named_msg(self, recp, msg): """Handler of a named msg command.""" try: name_hash = hash_string(recp) print('sending message to ' + recp + ' with hash ' + str(name_hash)) name_maintainer = self.local_.find_successor(name_hash) name_maintainer.user_command( 'named_msg', json.dumps({ 'name': recp, 'msg': 'message from ' + self.name_ + ": " + msg })) print('message sent') except socket.error: print("send message failed") def _accept_msg(self, msg): """Handler of a msg command.""" try: msg = json.loads(msg) if int(msg['key']) == self.local_.id() \ and self.msg_handler_ is not None: self.msg_handler_(msg['value']) else: print('received message not for us but for: ', msg['key']) except (KeyError, ValueError) as error: print("bad message accepted", error) return '' def _accept_migrate(self, msg): """Handler of a migrate command.""" try: msg = json.loads(msg) print("migrate accepted") migrate_channels = msg['channels'] for ch, ids in migrate_channels.items(): for id in ids: self._add_node_to_channel(ch, id) print('migrated channel ' + ch + ' with ' + str(len(ids)) + ' users.') migrate_names = msg['names'] for name, id in migrate_channels.items(): mntnd_names[name] = id print("migrate done") except (KeyError, ValueError) as error: print("bad migrate accepted", error) return '' def _add_node_to_channel(self, ch_name, id): if not (ch_name in self.mntnd_chnls): print('Creating new channel: ' + ch_name) self.mntnd_chnls[ch_name] = set() self.mntnd_chnls[ch_name].add(id) print('Node with id: ' + id + ' joined channel: ' + ch_name) def _accept_join(self, msg): """Handler of a join command.""" try: msg = json.loads(msg) self._add_node_to_channel(msg['channel_name'], msg['joining_id']) except (KeyError, ValueError) as error: print("bad join request accepted", error) return '' def _accept_leave(self, msg): """Handler of a leave command.""" try: msg = json.loads(msg) ch_name = msg['channel_name'] if not (ch_name in self.mntnd_chnls): print('This node is not maintainer of channel ' + ch_name + ", but leave command was received.") return '' id = msg['leaving_id'] if not (id in self.mntnd_chnls[ch_name]): print('User with id ' + id + ' is not member of channel ' + ch_name + ".") return '' self.mntnd_chnls[ch_name].remove(id) print('Node with id: ' + msg['leaving_id'] + ' left channel: ' + ch_name) except (KeyError, ValueError) as error: print("bad message accepted", error) return '' def _accept_channel_msg(self, msg): """Handler of a channel_msg command.""" try: msg = json.loads(msg) ch_name = msg['channel_name'] if not (ch_name in self.mntnd_chnls): print('This node is not maintainer of channel ' + ch_name + ", but send message command was received.") return '' for id in self.mntnd_chnls[ch_name]: try: node = self.local_.find_successor(int(id)) node.user_command( "msg", json.dumps({ 'key': id, 'value': msg['msg'] })) except socket.error: print("failed send of channel message to user " + id) except (KeyError, ValueError) as error: print("bad message accepted", error) return '' def send_msg(self, node_id, msg): """Attempts to send a message msg to a node with id node_id.""" try: node_id = node_id % SIZE str_hash = str(node_id) print("trying to send message to: " + str(node_id) \ + " with hash: " + str_hash) node = self.local_.find_successor(node_id) node.user_command( "msg", json.dumps({ 'key': str_hash, 'value': 'message from ' + self.name_ + ": " + msg })) print("message sent") except socket.error: print("send failed") def join_channel(self, ch_name): """ joins channel with name ch_name """ try: ch_hash = hash_string(ch_name) print("trying to join channel " + str(ch_name) + " with hash: " + str(ch_hash)) node = self.local_.find_successor(ch_hash) node.user_command( "join", json.dumps({ 'channel_name': ch_name, 'joining_id': str(self.local_.id()) })) self.joined_channels.add(ch_name) print("channel joined") except socket.error: print("channel join failed") def leave_channel(self, ch_name): """ leaves channel with name ch_name """ try: ch_hash = hash_string(ch_name) print("trying to leave channel " + str(ch_name) + " with hash: " + str(ch_hash)) node = self.local_.find_successor(ch_hash) node.user_command( "leave", json.dumps({ 'channel_name': ch_name, 'leaving_id': str(self.local_.id()) })) self.joined_channels.remove(ch_name) print("channel left") except socket.error: print("channel leave failed") def send_to_channel(self, ch_name, msg): try: ch_hash = hash_string(ch_name) print("sending message to channel " + str(ch_name) + " with hash: " + str(ch_hash)) node = self.local_.find_successor(ch_hash) node.user_command( "channel_msg", json.dumps({ 'channel_name': ch_name, 'msg': 'message from ' + self.name_ + ' to channel ' + ch_name + ': ' + msg })) print("message to channel sent") except socket.error: print("channel send failed") def _predecessor_change(self, new_predecessor): """Handler of the change of the predecessor event of the local chord node. :param new_predecessor: new predecessor of our node.""" if new_predecessor == self.local_: return migrate_chan = dict() for ch, ids in list(self.mntnd_chnls.items()): ch_id = hash_string(ch) if not self.local_.is_ours(ch_id): migrate_chan[ch] = list(ids) del self.mntnd_chnls[ch] print('migrating channel ' + ch + ' to a new predecessor.') migrate_names = dict() for name, id in list(self.mntnd_names.items()): name_id = hash_string(name) if not self.local_.is_ours(name_id): migrate_names[name] = id del self.mntnd_names[name] print('migrating name ' + name + ' to a new predecessor.') if len(migrate_chan) > 0 or len(migrate_names) > 0: migrate = {'channels': migrate_chan, 'names': migrate_names} new_predecessor.user_command('migrate', json.dumps(migrate)) def shutdown(self): """Shuts down the chat node.""" print("shutting down ...") name_hash = hash_string(self.name_) del_name_node = self.local_.find_successor(name_hash) del_name_node.user_command("delete_name", json.dumps({'name': self.name_})) for ch in list(self.joined_channels): self.leave_channel(ch) migrate_chan = dict() for ch, ids in list(self.mntnd_chnls.items()): if len(ids) > 0: migrate_chan[ch] = list(ids) print('migrating channel ' + ch + ' to a new predecessor.') if len(migrate_chan) > 0 or len(self.mntnd_names) > 0: migrate = {'channels': migrate_chan, 'names': self.mntnd_names} succ = self.local_.successor() succ.user_command('migrate', json.dumps(migrate)) self.local_.shutdown()
class DHT(object): def __init__(self, local_address, remote_address=None): self.local_ = Local(local_address, remote_address) def set_wrap(msg): return self._set(msg) def get_wrap(msg): return self._get(msg) self.data_ = {} self.shutdown_ = False self.local_.register_command("set", set_wrap) self.local_.register_command("get", get_wrap) self.daemons_ = {} self.daemons_['distribute_data'] = Daemon(self, 'distribute_data') self.daemons_['distribute_data'].start() self.local_.start() def shutdown(self): print("shutdown dht node") self.local_.shutdown() self.shutdown_ = True def _get(self, request): try: data = json.loads(request) # we have the key return json.dumps({'status': 'ok', 'data': self.get(data['key'])}) except Exception: # key not present return json.dumps({'status': 'failed'}) def _set(self, request): try: data = json.loads(request) key = data['key'] value = data['value'] self.set(key, value) return json.dumps({'status': 'ok'}) except Exception: # something is not working return json.dumps({'status': 'failed'}) def get(self, key): try: return self.data_[key] except Exception: # not in our range suc = self.local_.find_successor( int(hashlib.sha256(str(key).encode("utf-8")).hexdigest(), 16)) if self.local_.id() == suc.id(): # it's us but we don't have it return None try: response = suc.command('get %s' % json.dumps({'key': key})) if not response: raise Exception value = json.loads(response) if value['status'] != 'ok': raise Exception return value['data'] except Exception: return None def set(self, key, value): # eventually it will distribute the keys self.data_[key] = value @repeat_and_sleep(5) def distribute_data(self): to_remove = [] # to prevent from RTE in case data gets updated by other thread keys = list(self.data_.keys()) for key in keys: if self.local_.predecessor() and \ not inrange(int(hashlib.sha256(str(key).encode("utf-8")).hexdigest(),16), self.local_.predecessor().id(1), self.local_.id(1)): print( int( hashlib.sha256(str(key).encode("utf-8")).hexdigest(), 16)) try: node = self.local_.find_successor( int( hashlib.sha256( str(key).encode("utf-8")).hexdigest(), 16)) node.command("set %s" % json.dumps({ 'key': key, 'value': self.data_[key] })) # print "moved %s into %s" % (key, node.id()) to_remove.append(key) print("migrated") except OSError: print("error migrating") # we'll migrate it next time pass # remove all the keys we do not own any more for key in to_remove: del self.data_[key] # Keep calling us return True
class DHT(object): def __init__(self, local_address, remote_address=None): self.local_ = Local(local_address, remote_address) def set_wrap(msg): return self._set(msg) def get_wrap(msg): return self._get(msg) def backup_wrap(msg): return self._backup(msg) def dump_wrap(msg): return self.dump() def dumpall_wrap(msg): return self.dumpall() self.data_ = {} self.backup_ = {} self.shutdown_ = False self.backupFlag = False self.local_.register_command("set", set_wrap) self.local_.register_command("get", get_wrap) self.local_.register_command("backup", backup_wrap) self.local_.register_command("dump", dump_wrap) self.local_.register_command("dumpall", dumpall_wrap) self.local_.stabilize = self.stabilize self.local_.update_successors = self.update_successors self.daemons_ = {} self.daemons_['distribute_data'] = Daemon(self, 'distribute_data') self.daemons_['distribute_data'].start() self.daemons_['sendbackup'] = Daemon(self, 'sendbackup') self.daemons_['sendbackup'].start() self.local_.start() @repeat_and_sleep(STABILIZE_INT) @retry_on_socket_error(STABILIZE_RET) def stabilize(self): self.local_.log("stabilize") suc = self.local_.successor() # We may have found that x is our new successor iff # - x = pred(suc(n)) # - x exists # - x is in range (n, suc(n)) # - [n+1, suc(n)) is non-empty # fix finger_[0] if successor failed if suc.id() != self.local_.finger_[0].id(): self.local_.finger_[0] = suc x = suc.predecessor() if x != None and \ inrange(x.id(), self.local_.id(1), suc.id()) and \ self.local_.id(1) != suc.id() and \ x.ping(): self.local_.finger_[0] = x # We notify our new successor about us self.local_.successor().notify(self.local_) # Keep calling us for i in self.data_.iterkeys(): if self.backup_.has_key(i): self.backup_.pop(i) return True @repeat_and_sleep(UPDATE_SUCCESSORS_INT) @retry_on_socket_error(UPDATE_SUCCESSORS_RET) def update_successors(self): self.local_.log("update successor") suc = self.local_.successor() # if we are not alone in the ring, calculate if suc.id() != self.local_.id(): successors = [suc] suc_list = suc.get_successors() if suc_list and len(suc_list): successors += suc_list # if everything worked, we update if successors != self.local_.successors_: self.local_.successors_ = successors return True def shutdown(self): self.local_.shutdown() self.shutdown_ = True def _get(self, request): try: data = request # we have the key return json.dumps({'status': 'ok', 'data': self.get(data)}) except Exception: # key not present traceback.print_exc() return json.dumps({'status': 'failed'}) def _set(self, request): try: key = request.split(' ')[0] value = request.split(' ')[1] self.set(key, value) return json.dumps({'status': 'ok'}) # something is not working except Exception: traceback.print_exc() return json.dumps({'status': 'failed'}) def get(self, key): try: print self.local_.id() return self.data_[key] #['value'] except Exception: # not in our range suc = self.local_.find_successor(hash(key)) if self.local_.id() == suc.id(): # it's us but we don't have it if self.backup_.has_key(key): self.data_[key] = self.backup_[key] self.backupFlag = True print "Got at %s" % self.local_.id() return self.data_[key] #['value'] return None try: response = suc.command('get %s' % key) if not response: raise Exception value = json.loads(response) if value['status'] != 'ok': raise Exception return value['data'] except Exception: return None @repeat_and_sleep(4) def sendbackup(self): if self.backupFlag == False: pass else: temp = self.local_.successor() if self.data_ != {}: for i in [0, 1]: if temp.id() == self.local_.id(): break #while temp != self.local_: response = temp.command( 'backup %s' % json.dumps({ "id": i, "address": self.local_.address(), "data": self.data_ })) temp = temp.successor() print "Done backing up" self.backupFlag = False return True def setactual(self, suc, key, value): suc.command('set %s' % json.dumps({'key': key, 'value': value})) def set(self, key, value): # get successor for key self.data_[key] = value #{"value":value, 'backedUp':False} if key in self.backup_: del self.backup_[key] suc = self.local_.find_successor(hash(key)) if self.local_.id() == suc.id(): #its us #print "Stored at %s" % self.local_.id() self.backupFlag = True try: #self.data_[key]['backedUp']=True return True except Exception: print traceback.print_exc(Exception) return True def _backup(self, request): #print "Backing up: " + request + " at %s" % self.local_.id() try: data = json.loads(request) id = data['id'] address = data['address'] value = data['data'] self.backup(id, address, value) return json.dumps({'status': 'ok'}) # something is not working except Exception: traceback.print_exc() return json.dumps({'status': 'failed'}) def backup(self, id, address, value): #self.backup_ = {key: val for key, val in self.backup_.items() # if val['address'] != address} #self.backup_[id]={"address":address, "data":value} for key in value: self.backup_[key] = value[key] return True def printbackup(self): temp = "" for i in self.backup_: temp += '{"' + i.__str__ + '":"' + str(self.backup_[i]) + '"}' return temp def dump(self): dumpStr = "ID: %s" % self.local_.id() + "\n"\ + "Data: %s" % json.dumps(self.data_) + "\n"\ + "Backup: %s" % json.dumps(self.backup_) + "\n" return dumpStr def dumpall(self): retStr = self.dump() + "\n\n" temp = self.local_.successor() temp2 = None i = 0 while 1: temp2 = self.local_ retStr += temp.command("dump") + "\n\n" temp = temp.successor() #print str(temp.id()) + ":" +str(temp2.id()) if temp.id() == temp2.id(): break return retStr @repeat_and_sleep(5) def distribute_data(self): to_remove = [] # to prevent from RTE in case data gets updated by other thread keys = self.data_.keys() for key in keys: suc = self.local_.find_successor(hash(key)) if self.local_.id() != suc.id(): try: suc.command("set " + key + " " + self.data_[key]) # print "moved %s into %s" % (key, node.id()) to_remove.append(key) #print "migrated" except socket.error: print "error migrating" # we'll migrate it next time pass # remove all the keys we do not own any more if to_remove.__len__() != 0: self.backupFlag = True for key in to_remove: if key not in self.backup_: del self.data_[key] # Keep calling us return True
class DFS(object): def __init__(self, local_address, remote_address=None): self.local_ = Local(local_address, remote_address) def read_wrap(msg): return self._read(msg) def write_wrap(msg): return self._write(msg) def attr_wrap(msg): return self._attr(msg) self.data_ = {} self.attr_ = {} self.shutdown_ = False self.local_.register_command("read", read_wrap) self.local_.register_command("write", write_wrap) self.local_.register_command("attr", attr_wrap) self.local_.start() # helper function to eliminate duplicated code def get_offsets(self, offset, size): block_offset = offset / BLOCK_SIZE start = offset % BLOCK_SIZE end = min(start + size, BLOCK_SIZE) return (block_offset, start, end) def get_id(self, file_name, offset): block_offset, start, end = self.get_offsets(offset, 0) return "%s:%s" % (file_name, block_offset) def get_hash(self, file_name, offset): return hash(self.get_id(file_name, offset)) % settings.SIZE def get_remote(self, file_name, offset): hs = self.get_hash(file_name, offset) suc = self.local_.find_successor(hs) return suc def _read(self, request): # request = {'file_name':'my_file.txt', 'offset':<#NUMBER#>, 'size': <#NUMBER#>} # response = {'status':'failed'} | # {'status':'failed','code':<#CODE ERROR#>} | # {'status':'redirect'} # {'status':'ok','data':<#DATA READ AS B64#>} try: data = json.loads(request) if not self.local_.is_ours( self.get_hash(data['file_name'], data['offset'])): return json.dumps({'status': 'redirect'}) # otherwise continue result = self.read(data['file_name'], data['offset'], data['size']) if type(result) == type(-1): return json.dumps({'status': 'failed', 'code': result}) result = base64.b64encode(result) return json.dumpds({'status': 'ok', 'data': total_read}) except Exception: return json.dumps({'status': 'failed'}) def _write(self, request): # request = {'file_name':'my_file.txt', 'offset':<#NUMBER#>, 'data':<#B64 ENCODED DATA#>} # response = {'status':'failed'} | # {'status':'failed','code':<#CODE ERROR#>} | # {'status':'redirect'} # {'status':'ok','bytes':<#BYTES WROTE#>} try: data = json.loads(request) if not self.local_.is_ours( self.get_hash(data['file_name'], data['offset'])): return json.dumps({'status': 'redirect'}) result = self.write(data['file_name'], base64.b64decode(data['data']), offset) if result < 0: return json.dumps({'status': 'failed', 'code': result}) else: return json.dumps({'status': 'ok', 'bytes': result}) except Exception: return json.dumps({'status': 'failed'}) def _attr(self, request): # request = {'file_name':'my_file.txt'[,'size':<#NEW VALUE#>|,'mode':<#NEW MODE#>]} # response = {'status':'failed'} | # {'status':'failed','code':<#CODE ERROR#>} | # {'status':'redirect'} try: data = json.loads(request) if not self.local_.is_ours(self.get_hash(data['file_name'], 0)): return json.dumps({'status': 'redirect'}) except Exception: return json.dumps({'status': 'failed'}) def trunc_(self, request): pass def read(self, path, size, offset): attr = self.attr(file_name) if offset > attr.size: return "" block_offset, start, end = self.get_offsets(offset, size) block_id = self.get_id(file_name, offset) result = self.data_[block_id][start:end] def write(self, path, buf, offset): pass def attr(self, path): pass
class DHT(object): def __init__(self, local_address, remote_address = None): self.local_ = Local(local_address, remote_address) def set_wrap(msg): return self._set(msg) def get_wrap(msg): return self._get(msg) def backup_wrap(msg): return self._backup(msg) def dump_wrap(msg): return self.dump() def dumpall_wrap(msg): return self.dumpall() self.data_ = {} self.backup_ = {} self.shutdown_ = False self.backupFlag = False self.local_.register_command("set", set_wrap) self.local_.register_command("get", get_wrap) self.local_.register_command("backup", backup_wrap) self.local_.register_command("dump", dump_wrap) self.local_.register_command("dumpall", dumpall_wrap) self.local_.stabilize = self.stabilize self.local_.update_successors = self.update_successors self.daemons_ = {} self.daemons_['distribute_data'] = Daemon(self, 'distribute_data') self.daemons_['distribute_data'].start() self.daemons_['sendbackup'] = Daemon(self, 'sendbackup') self.daemons_['sendbackup'].start() self.local_.start() @repeat_and_sleep(STABILIZE_INT) @retry_on_socket_error(STABILIZE_RET) def stabilize(self): self.local_.log("stabilize") suc = self.local_.successor() # We may have found that x is our new successor iff # - x = pred(suc(n)) # - x exists # - x is in range (n, suc(n)) # - [n+1, suc(n)) is non-empty # fix finger_[0] if successor failed if suc.id() != self.local_.finger_[0].id(): self.local_.finger_[0] = suc x = suc.predecessor() if x != None and \ inrange(x.id(), self.local_.id(1), suc.id()) and \ self.local_.id(1) != suc.id() and \ x.ping(): self.local_.finger_[0] = x # We notify our new successor about us self.local_.successor().notify(self.local_) # Keep calling us for i in self.data_.iterkeys(): if self.backup_.has_key(i): self.backup_.pop(i) return True @repeat_and_sleep(UPDATE_SUCCESSORS_INT) @retry_on_socket_error(UPDATE_SUCCESSORS_RET) def update_successors(self): self.local_.log("update successor") suc = self.local_.successor() # if we are not alone in the ring, calculate if suc.id() != self.local_.id(): successors = [suc] suc_list = suc.get_successors() if suc_list and len(suc_list): successors += suc_list # if everything worked, we update if successors!=self.local_.successors_: self.local_.successors_ = successors return True def shutdown(self): self.local_.shutdown() self.shutdown_ = True def _get(self, request): try: data = request # we have the key return json.dumps({'status':'ok', 'data':self.get(data)}) except Exception: # key not present traceback.print_exc() return json.dumps({'status':'failed'}) def _set(self, request): try: key = request.split(' ')[0] value = request.split(' ')[1] self.set(key, value) return json.dumps({'status':'ok'}) # something is not working except Exception: traceback.print_exc() return json.dumps({'status':'failed'}) def get(self, key): try: print self.local_.id() return self.data_[key]#['value'] except Exception: # not in our range suc = self.local_.find_successor(hash(key)) if self.local_.id() == suc.id(): # it's us but we don't have it if self.backup_.has_key(key): self.data_[key] = self.backup_[key] self.backupFlag = True print "Got at %s" % self.local_.id() return self.data_[key]#['value'] return None try: response = suc.command('get %s' % key) if not response: raise Exception value = json.loads(response) if value['status'] != 'ok': raise Exception return value['data'] except Exception: return None @repeat_and_sleep(4) def sendbackup(self): if self.backupFlag==False: pass else: temp = self.local_.successor() if self.data_ != {}: for i in [0,1]: if temp.id() == self.local_.id(): break #while temp != self.local_: response = temp.command('backup %s' % json.dumps({"id":i,"address":self.local_.address(), "data":self.data_})) temp = temp.successor() print "Done backing up" self.backupFlag = False return True def setactual(self, suc, key, value): suc.command('set %s' % json.dumps({'key':key,'value':value})) def set(self, key, value): # get successor for key self.data_[key]=value#{"value":value, 'backedUp':False} if key in self.backup_: del self.backup_[key] suc = self.local_.find_successor(hash(key)) if self.local_.id() == suc.id(): #its us #print "Stored at %s" % self.local_.id() self.backupFlag = True try: #self.data_[key]['backedUp']=True return True except Exception: print traceback.print_exc(Exception) return True def _backup(self, request): #print "Backing up: " + request + " at %s" % self.local_.id() try: data = json.loads(request) id = data['id'] address = data['address'] value = data['data'] self.backup(id, address, value) return json.dumps({'status':'ok'}) # something is not working except Exception: traceback.print_exc() return json.dumps({'status':'failed'}) def backup(self, id, address, value): #self.backup_ = {key: val for key, val in self.backup_.items() # if val['address'] != address} #self.backup_[id]={"address":address, "data":value} for key in value: self.backup_[key]=value[key] return True def printbackup(self): temp = "" for i in self.backup_: temp += '{"'+i.__str__+'":"'+str(self.backup_[i])+'"}' return temp def dump(self): dumpStr = "ID: %s" % self.local_.id() + "\n"\ + "Data: %s" % json.dumps(self.data_) + "\n"\ + "Backup: %s" % json.dumps(self.backup_) + "\n" return dumpStr def dumpall(self): retStr = self.dump() + "\n\n" temp = self.local_.successor() temp2 = None i = 0 while 1: temp2 = self.local_ retStr+=temp.command("dump")+"\n\n" temp = temp.successor() #print str(temp.id()) + ":" +str(temp2.id()) if temp.id()==temp2.id(): break return retStr @repeat_and_sleep(5) def distribute_data(self): to_remove = [] # to prevent from RTE in case data gets updated by other thread keys = self.data_.keys() for key in keys: suc = self.local_.find_successor(hash(key)) if self.local_.id() != suc.id(): try: suc.command("set "+key+" "+self.data_[key]) # print "moved %s into %s" % (key, node.id()) to_remove.append(key) #print "migrated" except socket.error: print "error migrating" # we'll migrate it next time pass # remove all the keys we do not own any more if to_remove.__len__()!=0: self.backupFlag = True for key in to_remove: if key not in self.backup_: del self.data_[key] # Keep calling us return True
class DHT(object): def __init__(self, local_address, remote_address = None): self.local_ = Local(local_address, remote_address) def set_wrap(msg): return self._set(msg) def get_wrap(msg): return self._get(msg) self.data_ = {} self.shutdown_ = False self.local_.register_command("set", set_wrap) self.local_.register_command("get", get_wrap) self.daemons_ = {} self.daemons_['distribute_data'] = Daemon(self, 'distribute_data') self.daemons_['distribute_data'].start() self.local_.start() def shutdown(self): self.local_.shutdown() self.shutdown_ = True def _get(self, request): try: data = json.loads(request) # we have the key return json.dumps({'status':'ok', 'data':self.get(data['key'])}) except Exception: # key not present return json.dumps({'status':'failed'}) def _set(self, request): try: data = json.loads(request) key = data['key'] value = data['value'] self.set(key, value) return json.dumps({'status':'ok'}) except Exception: # something is not working return json.dumps({'status':'failed'}) def get(self, key): try: return self.data_[key] except Exception: # not in our range suc = self.local_.find_successor(hash(key)) if self.local_.id() == suc.id(): # it's us but we don't have it return None try: response = suc.command('get %s' % json.dumps({'key':key})) if not response: raise Exception value = json.loads(response) if value['status'] != 'ok': raise Exception return value['data'] except Exception: return None def set(self, key, value): # eventually it will distribute the keys self.data_[key] = value @repeat_and_sleep(5) def distribute_data(self): to_remove = [] # to prevent from RTE in case data gets updated by other thread keys = self.data_.keys() for key in keys: if self.local_.predecessor() and \ not inrange(hash(key), self.local_.predecessor().id(1), self.local_.id(1)): try: node = self.local_.find_successor(hash(key)) node.command("set %s" % json.dumps({'key':key, 'value':self.data_[key]})) # print "moved %s into %s" % (key, node.id()) to_remove.append(key) print "migrated" except socket.error: print "error migrating" # we'll migrate it next time pass # remove all the keys we do not own any more for key in to_remove: del self.data_[key] # Keep calling us return True
class DHT(object): def __init__(self, local_address, remote_address=None): self.local_ = Local(local_address, remote_address) def set_wrap(msg): return self._set(msg) def get_wrap(msg): return self._get(msg) def post_wrap(msg): return self._post(msg) def poll_wrap(msg): return self._poll(msg) self.data_ = {} self.channels = {} self.shutdown_ = False self.local_.register_command("set", set_wrap) self.local_.register_command("get", get_wrap) self.local_.register_command("post", post_wrap) self.local_.register_command("poll", poll_wrap) self.daemons_ = {} self.daemons_['distribute_data'] = Daemon(self, 'distribute_data') self.daemons_['distribute_data'].start() self.local_.start() def shutdown(self): self.local_.shutdown() self.shutdown_ = True def _get(self, request): try: data = json.loads(request) # we have the key return json.dumps({'status': 'ok', 'data': self.get(data['key'])}) except Exception: # key not present return json.dumps({'status': 'failed'}) def _poll(self, request): try: data = json.loads(request) key = data['key'] time = data['time'] # we have the key #print(("polling",key,time)) return json.dumps({ 'status': 'ok', 'data': self.pollChan(key, time) }) except Exception: # key not present return json.dumps({'status': 'failed'}) def _post(self, request): try: data = json.loads(request) key = data['key'] value = data['value'] # we have the key self.postChan(key, value) return json.dumps({'status': 'ok'}) except Exception: # key not present return json.dumps({'status': 'failed'}) def _set(self, request): try: data = json.loads(request) key = data['key'] value = data['value'] self.set(key, value) return json.dumps({'status': 'ok'}) except Exception: # something is not working return json.dumps({'status': 'failed'}) def pollChan(self, key, timestamp): if key in self.channels.keys(): tmp = self.channels[key] output = list(filter(tmp, lambda x: x[0] > timestamp)) return output else: return [] def postChan(self, key, msg): now = time.time() if key in list(self.channels.keys()): self.channels[key].append((now, msg)) else: self.channels[key] = [(now, msg)] def get(self, key): try: return self.data_[key] except Exception: # not in our range suc = self.local_.find_successor(hash(key)) if self.local_.id() == suc.id(): # it's us but we don't have it return None try: response = suc.command('get %s' % json.dumps({'key': key})) if not response: raise Exception value = json.loads(response) if value['status'] != 'ok': raise Exception return value['data'] except Exception: return None def post(self, key, value): # not in our range suc = self.local_.find_successor(hash(key)) if self.local_.id() == suc.id(): # it's us but we don't have it self.postChan(key, value) return True try: response = suc.command('post %s' % json.dumps({ 'key': key, 'value': value })) if not response: raise Exception value = json.loads(response) if value['status'] != 'ok': raise Exception return True except Exception: return False def poll(self, key, time): #print("polling") suc = self.local_.find_successor(hash(key)) if self.local_.id() == suc.id(): return self.pollChan(key, time) else: # not in our range suc = self.local_.find_successor(hash(key)) if self.local_.id() == suc.id(): # it's us but we don't have it return None response = suc.command('poll %s' % json.dumps({ 'key': key, 'time': time })) if not response: raise Exception value = json.loads(response) if value['status'] != 'ok': print(value) return value['data'] """ except Exception as e: print(e) return None """ def set(self, key, value): # eventually it will distribute the keys self.data_[key] = value @repeat_and_sleep(5) def distribute_data(self): to_remove = [] # to prevent from RTE in case data gets updated by other thread keys = list(self.data_.keys()) for key in keys: if self.local_.predecessor() and \ not inrange(hash(key), self.local_.predecessor().id(1), self.local_.id(1)): try: node = self.local_.find_successor(hash(key)) node.command("set %s" % json.dumps({ 'key': key, 'value': self.data_[key] })) # print "moved %s into %s" % (key, node.id()) to_remove.append(key) print("migrated") except socket.error: print("error migrating") # we'll migrate it next time pass # remove all the keys we do not own any more for key in to_remove: del self.data_[key] # Keep calling us return True