def listx(self, tag_list): tag_id_list = [hash_str(x) for x in tag_list] file_list = None for tag, tag_id in zip(tag_list, tag_id_list): node = self.dht.find_successor(tag_id) node_ro = get_remote(node) node_storage = node_ro.get_remote_storage() node_storage_ro = get_remote(node_storage) files = node_storage_ro.get_files_of_tag(tag_id) if file_list is None: file_list = files else: file_list.intersection_update(files) file_names = {} for file_id in file_list: node = self.dht.find_successor(file_id) node_ro = get_remote(node) node_storage = node_ro.get_remote_storage() node_storage_ro = get_remote(node_storage) #print([file_id]) file = node_storage_ro.get_real_name(file_id) #print('ok2') if file is not None: file_names[file] = node_storage_ro.get_tags_of_file(file) # dame el nombre del archivo y guardalo en file_names por denis return file_names
def notify(self, other: RemoteObj): """ <other> might be our predecessor """ p = self.predecessor() if p is None: self.set_predecessor(other) other_ro = get_remote(other) other_storage = other_ro.get_remote_storage() # send all my keys between self.id and other.id to other self.storage.release_tags(self.id, other_storage) self.storage.release_file_tags(self.id, other_storage) self.storage.release_files(self.id, other_storage) elif inrange(other.id, p.id, self.id): self.set_predecessor(other) other_ro = get_remote(other) other_storage = other_ro.get_remote_storage() # send all my keys between p.id and other.id to other self.storage.release_tags(p.id, other_storage) self.storage.release_file_tags(p.id, other_storage) self.storage.release_files(p.id, other_storage) elif not self.ping(p): self.set_predecessor(other) # load in storage all key from backups between other.id p.id self.storage.load_from_tags_backup(other.id, p.id) self.storage.load_from_file_tags_backup(other.id, p.id) self.storage.load_from_files_backup(other.id, p.id)
def add(self, file_path_list: list, tag_list: list, ip: str, port: int): file_list = [x.split('/')[-1] for x in file_path_list] file_id_list = [hash_str(x) for x in file_list] tag_id_list = [hash_str(x) for x in tag_list] for file_path, file, file_id in zip(file_path_list, file_list, file_id_list): node = self.dht.find_successor(file_id) node_ro = get_remote(node) node_storage = node_ro.get_remote_storage() node_storage_ro = get_remote(node_storage) #print([file], [ip], [port]) #print(node_storage_ro) node_storage_ro.upload(file, file_path, ip, port) # upload file node_storage_ro.add_file_to_tags(file, tag_list) #print('ok') for tag, tag_id in zip(tag_list, tag_id_list): node = self.dht.find_successor(tag_id) node_ro = get_remote(node) node_storage = node_ro.get_remote_storage() node_storage_ro = get_remote(node_storage) node_storage_ro.add_tag_to_files(tag_id, file_id_list)
def download(self, file_name): file_id = hash_str(file_name) node = self.dht.find_successor(file_id) node_ro = get_remote(node) node_storage = node_ro.get_remote_storage() node_storage_ro = get_remote(node_storage) data = node_storage_ro.download(file_id) for d in data: yield d
def join(self, remote_obj=None) -> True: self._finger = [None] * M self._predecessor = None self.successor_list = [] if remote_obj is None: try: ns = Pyro4.locateNS() for server in ns.list(): if "NameServer" not in server: try: with Pyro4.Proxy(ns.list()[server]) as p: remote_info = p.get_info() remote_obj = (remote_info.ip, remote_info.port) break except: pass except Exception as x: print(x) if remote_obj is None: self.set_successor(self.get_info()) return True #print(f'Trying connect to {remote_obj}') other = remote_obj if isinstance(remote_obj, RemoteObj) else RemoteObj( -1, *remote_obj) if self.ping(other): self.set_successor(get_remote(other).find_successor(self.id)) return True return False
def _find_successor(self, id: int) -> RemoteObj: """ Ask node to find id’s successor """ p = self.find_predecessor(id) ro = get_remote(p) return ro.successor()
def send_files_backup_copy_to(self, remote_storage): try: remote_storage_ro = get_remote(remote_storage) #for file in self.files_backups.keys(): for file in self.files.keys(): remote_storage_ro.upload_to_backup(self.files[file], self.ip, self.port) except Exception as ex: print(ex)
def stabilize(self): """ Periodically verify inmediate successor, and tell the successor about this node """ if not self.ping(self.successor_list[0]): return s = self.successor() s_ro = get_remote(s) x = s_ro.predecessor() if x is not None and inrange(x.id, self.id, s.id): self.set_successor(x) s = self.successor() get_remote(s).notify(self.get_info())
def release_tags(self, p_id: int, remote_storage): release_keys = [key for key in self.tags.keys() if inrange(key, p_id, remote_storage.id, (False, True))] print(f'Releasing tags {release_keys} to {remote_storage.id}') try: remote_storage_ro = get_remote(remote_storage) for tag in release_keys: file_list = self.tags.pop(tag) remote_storage_ro.add_tag_to_files(tag, file_list) except Exception as ex: print(ex)
def release_file_tags(self, p_id: int, remote_storage): release_keys = [key for key in self.file_tags.keys() if inrange(hash_str(key), p_id, remote_storage.id, (False, True))] print(f'Releasing files {release_keys} to {remote_storage.id}') try: remote_storage_ro = get_remote(remote_storage) for file in release_keys: tag_list = self.file_tags.pop(file) remote_storage_ro.add_file_to_tags(file, tag_list) except Exception as ex: print(ex)
def delete(self, tag_list): tag_id_list = [hash_str(x) for x in tag_list] file_list = None for tag, tag_id in zip(tag_list, tag_id_list): node = self.dht.find_successor(tag_id) node_ro = get_remote(node) node_storage = node_ro.get_remote_storage() node_storage_ro = get_remote(node_storage) files = node_storage_ro.get_files_of_tag(tag_id) if file_list is None: file_list = files else: file_list.intersection_update(files) for file_id in file_list: node = self.dht.find_successor(file_id) node_ro = get_remote(node) node_storage = node_ro.get_remote_storage() node_storage_ro = get_remote(node_storage) #print([file_id]) node_storage_ro.del_files([file_id]) # del files #print('ok1') for tag, tag_id in zip(tag_list, tag_id_list): node = self.dht.find_successor(tag_id) node_ro = get_remote(node) node_storage = node_ro.get_remote_storage() node_storage_ro = get_remote(node_storage) node_storage_ro.remove_files_of_tag(tag_id, file_list)
def delete_tags(self, tag_list, tag_list2): tag_id_list = [hash_str(x) for x in tag_list] tag_id_list2 = [hash_str(x) for x in tag_list2] file_list = None for tag, tag_id in zip(tag_list, tag_id_list): node = self.dht.find_successor(tag_id) node_ro = get_remote(node) node_storage = node_ro.get_remote_storage() node_storage_ro = get_remote(node_storage) files = node_storage_ro.get_files_of_tag(tag_id) if file_list is None: file_list = files else: file_list.intersection_update(files) for file_id in file_list: node = self.dht.find_successor(file_id) node_ro = get_remote(node) node_storage = node_ro.get_remote_storage() node_storage_ro = get_remote(node_storage) node_storage_ro.remove_tags_of_file(file_id, tag_list2) for tag2, tag_id2 in zip(tag_list2, tag_id_list2): node = self.dht.find_successor(tag_id2) node_ro = get_remote(node) node_storage = node_ro.get_remote_storage() node_storage_ro = get_remote(node_storage) node_storage_ro.remove_files_of_tag(tag_id2, file_list)
def release_files(self, p_id: int, remote_storage): release_keys = [key for key in self.files.keys() if inrange(key, p_id, remote_storage.id, (False, True))] print(f'Releasing files {release_keys} to {remote_storage.id}') try: remote_storage_ro = get_remote(remote_storage) for file in release_keys: realname = self.files.pop(file) self.files_backups[file] = realname remote_storage_ro.upload_to_files(realname, self.ip, self.port) except Exception as ex: print(ex)
def find_predecessor(self, id: int) -> RemoteObj: """ Ask node to find id’s predecessor """ # corner case # if self.id() == id: # return self.predecessor() p = self.get_info() p_ro = get_remote(p) while not inrange(id, p.id, p_ro.successor().id, (False, True)): np = p_ro.closest_preceding_finger(id) # corner case # if p.id() == np.id(): # break p = np p_ro = get_remote(p) return p
def ping(self, other: RemoteObj) -> bool: print( f'ID: {self.id} SUCC: {None if self._finger[0] is None else self._finger[0].id} SUCC-LIST: {[None if x is None else x.id for x in self.successor_list]}' ) if other is None: return False try: # print(f'Try Ping: {self} to {other}') #print() ro = get_remote(other) return ro.pong() except Exception as ex: print(f'Ping: {ex}') return False
def send_files_copy_to(self, remote_storage): try: remote_storage_ro = get_remote(remote_storage) remote_storage_ro.update_file_tags_backup(self.file_tags) except Exception as ex: print(ex)
def update_successors(self): # for i, s in enumerate(self.successor_list): # if self.ping(s): # if i > 0: # s_ro = get_remote(s) # s_ro.notify(self.get_info()) # self.successor_list = [s] # break # else: # for i in range(1, M): # finger = self.get_finger(i) # if self.ping(finger) and finger.id != self.id: # finger_ro = get_remote(finger) # succ = finger_ro.find_successor(self.id) # succ_ro = get_remote(succ) # succ_ro.notify(self.get_info()) # self.successor_list = [succ] # break # self.successor_list = [succ for succ in self.successor_list if self.ping(succ)] # if rd.random() <= 1/6 or len(self.successor_list) == 0: # if len(self.successor_list) == 0: # succ = self.successor() # succ_ro = get_remote(succ) # n_succ = succ_ro.find_successor(self.id) if succ.id != self.id else self._find_successor(self.id) # self.successor_list = [n_succ] # if len(self.successor_list) < M: # l_succ = self.successor_list[-1] # l_succ_ro = get_remote(l_succ) # nl_succ = l_succ_ro.successor() # if nl_succ.id != self.id and l_succ.id != nl_succ.id: # self.successor_list.append(nl_succ) # nl_succ_ro = get_remote(nl_succ) # # sending replication # nl_succ_storage = nl_succ_ro.get_remote_storage() # # succ = self.successor() # self.set_successor(succ) # succ_ro = get_remote(succ) # succ_ro.notify(self.get_info()) succ = self.successor() if succ.id != self.id: successors = [succ] suc_ro = get_remote(succ) suc_list = suc_ro.get_successor_list()[:M - 1] if suc_list and len(suc_list): successors += suc_list self.successor_list = successors # replicating copys = set(successors[:3]) for copy in copys: copy_ro = get_remote(copy) copy_storage = copy_ro.get_remote_storage() self.storage.send_tags_copy_to(copy_storage) self.storage.send_files_copy_to(copy_storage) self.storage.send_files_backup_copy_to(copy_storage) self.set_successor(succ)