예제 #1
0
    def listx(self, tag_list):
        tag_id_list = [hash_str(x) for x in tag_list]

        file_list = None

        for tag, tag_id in zip(tag_list, tag_id_list):
            node = self.dht.find_successor(tag_id)
            node_ro = get_remote(node)
            node_storage = node_ro.get_remote_storage()
            node_storage_ro = get_remote(node_storage)

            files = node_storage_ro.get_files_of_tag(tag_id)

            if file_list is None:
                file_list = files
            else:
                file_list.intersection_update(files)

        file_names = {}

        for file_id in file_list:
            node = self.dht.find_successor(file_id)
            node_ro = get_remote(node)
            node_storage = node_ro.get_remote_storage()
            node_storage_ro = get_remote(node_storage)

            #print([file_id])
            file = node_storage_ro.get_real_name(file_id)
            #print('ok2')
            if file is not None:
                file_names[file] = node_storage_ro.get_tags_of_file(file)
            # dame el nombre del archivo y guardalo en file_names por denis

        return file_names
예제 #2
0
    def notify(self, other: RemoteObj):
        """
        <other> might be our predecessor
        """

        p = self.predecessor()

        if p is None:
            self.set_predecessor(other)
            other_ro = get_remote(other)
            other_storage = other_ro.get_remote_storage()
            # send all my keys between self.id and other.id to other
            self.storage.release_tags(self.id, other_storage)
            self.storage.release_file_tags(self.id, other_storage)
            self.storage.release_files(self.id, other_storage)
        elif inrange(other.id, p.id, self.id):
            self.set_predecessor(other)
            other_ro = get_remote(other)
            other_storage = other_ro.get_remote_storage()
            # send all my keys between p.id and other.id to other
            self.storage.release_tags(p.id, other_storage)
            self.storage.release_file_tags(p.id, other_storage)
            self.storage.release_files(p.id, other_storage)
        elif not self.ping(p):
            self.set_predecessor(other)
            # load in storage all key from backups between other.id p.id
            self.storage.load_from_tags_backup(other.id, p.id)
            self.storage.load_from_file_tags_backup(other.id, p.id)
            self.storage.load_from_files_backup(other.id, p.id)
예제 #3
0
    def add(self, file_path_list: list, tag_list: list, ip: str, port: int):
        file_list = [x.split('/')[-1] for x in file_path_list]
        file_id_list = [hash_str(x) for x in file_list]
        tag_id_list = [hash_str(x) for x in tag_list]

        for file_path, file, file_id in zip(file_path_list, file_list,
                                            file_id_list):
            node = self.dht.find_successor(file_id)
            node_ro = get_remote(node)
            node_storage = node_ro.get_remote_storage()
            node_storage_ro = get_remote(node_storage)

            #print([file], [ip], [port])
            #print(node_storage_ro)
            node_storage_ro.upload(file, file_path, ip, port)  # upload file
            node_storage_ro.add_file_to_tags(file, tag_list)
            #print('ok')

        for tag, tag_id in zip(tag_list, tag_id_list):
            node = self.dht.find_successor(tag_id)
            node_ro = get_remote(node)
            node_storage = node_ro.get_remote_storage()
            node_storage_ro = get_remote(node_storage)

            node_storage_ro.add_tag_to_files(tag_id, file_id_list)
예제 #4
0
 def download(self, file_name):
     file_id = hash_str(file_name)
     node = self.dht.find_successor(file_id)
     node_ro = get_remote(node)
     node_storage = node_ro.get_remote_storage()
     node_storage_ro = get_remote(node_storage)
     data = node_storage_ro.download(file_id)
     for d in data:
         yield d
예제 #5
0
    def join(self, remote_obj=None) -> True:
        self._finger = [None] * M
        self._predecessor = None
        self.successor_list = []

        if remote_obj is None:
            try:
                ns = Pyro4.locateNS()
                for server in ns.list():
                    if "NameServer" not in server:
                        try:
                            with Pyro4.Proxy(ns.list()[server]) as p:
                                remote_info = p.get_info()
                                remote_obj = (remote_info.ip, remote_info.port)
                            break
                        except:
                            pass
            except Exception as x:
                print(x)
            if remote_obj is None:
                self.set_successor(self.get_info())
                return True

        #print(f'Trying connect to {remote_obj}')
        other = remote_obj if isinstance(remote_obj, RemoteObj) else RemoteObj(
            -1, *remote_obj)
        if self.ping(other):
            self.set_successor(get_remote(other).find_successor(self.id))
            return True

        return False
예제 #6
0
    def _find_successor(self, id: int) -> RemoteObj:
        """
        Ask node to find id’s successor
        """

        p = self.find_predecessor(id)
        ro = get_remote(p)
        return ro.successor()
 def send_files_backup_copy_to(self, remote_storage):
     try:
         remote_storage_ro = get_remote(remote_storage)
         #for file in self.files_backups.keys():
         for file in self.files.keys():
             remote_storage_ro.upload_to_backup(self.files[file], self.ip, self.port)
     except Exception as ex:
         print(ex)
예제 #8
0
    def stabilize(self):
        """
        Periodically verify inmediate successor,
        and tell the successor about this node
        """

        if not self.ping(self.successor_list[0]):
            return

        s = self.successor()
        s_ro = get_remote(s)
        x = s_ro.predecessor()

        if x is not None and inrange(x.id, self.id, s.id):
            self.set_successor(x)

        s = self.successor()
        get_remote(s).notify(self.get_info())
 def release_tags(self, p_id: int, remote_storage):
     release_keys = [key for key in self.tags.keys() if inrange(key, p_id, remote_storage.id, (False, True))]
     
     print(f'Releasing tags {release_keys} to {remote_storage.id}')
     
     try:
         remote_storage_ro = get_remote(remote_storage)
         for tag in release_keys:
             file_list = self.tags.pop(tag)
             remote_storage_ro.add_tag_to_files(tag, file_list)
     except Exception as ex:
         print(ex)
 def release_file_tags(self, p_id: int, remote_storage):
     release_keys = [key for key in self.file_tags.keys() if inrange(hash_str(key), p_id, remote_storage.id, (False, True))]
     
     print(f'Releasing files {release_keys} to {remote_storage.id}')
     
     try:
         remote_storage_ro = get_remote(remote_storage)
         for file in release_keys:
             tag_list = self.file_tags.pop(file)
             remote_storage_ro.add_file_to_tags(file, tag_list)
     except Exception as ex:
         print(ex)
예제 #11
0
    def delete(self, tag_list):
        tag_id_list = [hash_str(x) for x in tag_list]

        file_list = None

        for tag, tag_id in zip(tag_list, tag_id_list):
            node = self.dht.find_successor(tag_id)
            node_ro = get_remote(node)
            node_storage = node_ro.get_remote_storage()
            node_storage_ro = get_remote(node_storage)

            files = node_storage_ro.get_files_of_tag(tag_id)

            if file_list is None:
                file_list = files
            else:
                file_list.intersection_update(files)

        for file_id in file_list:
            node = self.dht.find_successor(file_id)
            node_ro = get_remote(node)
            node_storage = node_ro.get_remote_storage()
            node_storage_ro = get_remote(node_storage)

            #print([file_id])
            node_storage_ro.del_files([file_id])  # del files
            #print('ok1')

        for tag, tag_id in zip(tag_list, tag_id_list):
            node = self.dht.find_successor(tag_id)
            node_ro = get_remote(node)
            node_storage = node_ro.get_remote_storage()
            node_storage_ro = get_remote(node_storage)

            node_storage_ro.remove_files_of_tag(tag_id, file_list)
예제 #12
0
    def delete_tags(self, tag_list, tag_list2):
        tag_id_list = [hash_str(x) for x in tag_list]
        tag_id_list2 = [hash_str(x) for x in tag_list2]

        file_list = None

        for tag, tag_id in zip(tag_list, tag_id_list):
            node = self.dht.find_successor(tag_id)
            node_ro = get_remote(node)
            node_storage = node_ro.get_remote_storage()
            node_storage_ro = get_remote(node_storage)

            files = node_storage_ro.get_files_of_tag(tag_id)

            if file_list is None:
                file_list = files
            else:
                file_list.intersection_update(files)

        for file_id in file_list:
            node = self.dht.find_successor(file_id)
            node_ro = get_remote(node)
            node_storage = node_ro.get_remote_storage()
            node_storage_ro = get_remote(node_storage)

            node_storage_ro.remove_tags_of_file(file_id, tag_list2)

        for tag2, tag_id2 in zip(tag_list2, tag_id_list2):
            node = self.dht.find_successor(tag_id2)
            node_ro = get_remote(node)
            node_storage = node_ro.get_remote_storage()
            node_storage_ro = get_remote(node_storage)

            node_storage_ro.remove_files_of_tag(tag_id2, file_list)
 def release_files(self, p_id: int, remote_storage):
     release_keys = [key for key in self.files.keys() if inrange(key, p_id, remote_storage.id, (False, True))]
     
     print(f'Releasing files {release_keys} to {remote_storage.id}')
     
     try:
         remote_storage_ro = get_remote(remote_storage)
         for file in release_keys:
             realname = self.files.pop(file)
             self.files_backups[file] = realname
             remote_storage_ro.upload_to_files(realname, self.ip, self.port)
     except Exception as ex:
         print(ex)
예제 #14
0
    def find_predecessor(self, id: int) -> RemoteObj:
        """
        Ask node to find id’s predecessor
        """

        # corner case
        # if self.id() == id:
        #     return self.predecessor()

        p = self.get_info()
        p_ro = get_remote(p)

        while not inrange(id, p.id, p_ro.successor().id, (False, True)):

            np = p_ro.closest_preceding_finger(id)

            # corner case
            # if p.id() == np.id():
            #     break

            p = np
            p_ro = get_remote(p)

        return p
예제 #15
0
    def ping(self, other: RemoteObj) -> bool:
        print(
            f'ID: {self.id} SUCC: {None if self._finger[0] is None else self._finger[0].id} SUCC-LIST: {[None if x is None else x.id for x in self.successor_list]}'
        )

        if other is None:
            return False

        try:
            # print(f'Try Ping: {self} to {other}')
            #print()
            ro = get_remote(other)
            return ro.pong()
        except Exception as ex:
            print(f'Ping: {ex}')
            return False
 def send_files_copy_to(self, remote_storage):
     try:
         remote_storage_ro = get_remote(remote_storage)
         remote_storage_ro.update_file_tags_backup(self.file_tags)
     except Exception as ex:
         print(ex)
예제 #17
0
    def update_successors(self):
        # for i, s in enumerate(self.successor_list):
        #     if self.ping(s):
        #         if i > 0:
        #             s_ro = get_remote(s)
        #             s_ro.notify(self.get_info())
        #             self.successor_list = [s]

        #         break
        # else:
        #     for i in range(1, M):
        #         finger = self.get_finger(i)
        #         if self.ping(finger) and finger.id != self.id:
        #             finger_ro = get_remote(finger)
        #             succ = finger_ro.find_successor(self.id)

        #             succ_ro = get_remote(succ)
        #             succ_ro.notify(self.get_info())
        #             self.successor_list = [succ]

        #             break

        # self.successor_list = [succ for succ in self.successor_list if self.ping(succ)]

        # if rd.random() <= 1/6 or len(self.successor_list) == 0:
        # if len(self.successor_list) == 0:
        #     succ = self.successor()
        #     succ_ro = get_remote(succ)
        #     n_succ = succ_ro.find_successor(self.id) if succ.id != self.id else self._find_successor(self.id)
        #     self.successor_list = [n_succ]

        # if len(self.successor_list) < M:
        #     l_succ = self.successor_list[-1]
        #     l_succ_ro = get_remote(l_succ)
        #     nl_succ = l_succ_ro.successor()

        #     if nl_succ.id != self.id and l_succ.id != nl_succ.id:
        #         self.successor_list.append(nl_succ)
        #         nl_succ_ro = get_remote(nl_succ)
        #         # sending replication
        #         nl_succ_storage = nl_succ_ro.get_remote_storage()
        #

        # succ = self.successor()
        # self.set_successor(succ)
        # succ_ro = get_remote(succ)
        # succ_ro.notify(self.get_info())

        succ = self.successor()

        if succ.id != self.id:
            successors = [succ]
            suc_ro = get_remote(succ)
            suc_list = suc_ro.get_successor_list()[:M - 1]

            if suc_list and len(suc_list):
                successors += suc_list

            self.successor_list = successors

            # replicating
            copys = set(successors[:3])
            for copy in copys:
                copy_ro = get_remote(copy)
                copy_storage = copy_ro.get_remote_storage()
                self.storage.send_tags_copy_to(copy_storage)
                self.storage.send_files_copy_to(copy_storage)
                self.storage.send_files_backup_copy_to(copy_storage)

        self.set_successor(succ)