def listx(self, tag_list):
tag_id_list = [hash_str(x) for x in tag_list]
file_list = None
for tag, tag_id in zip(tag_list, tag_id_list):
node = self.dht.find_successor(tag_id)
node_ro = get_remote(node)
node_storage = node_ro.get_remote_storage()
node_storage_ro = get_remote(node_storage)
files = node_storage_ro.get_files_of_tag(tag_id)
if file_list is None:
file_list = files
else:
file_list.intersection_update(files)
file_names = {}
for file_id in file_list:
node = self.dht.find_successor(file_id)
node_ro = get_remote(node)
node_storage = node_ro.get_remote_storage()
node_storage_ro = get_remote(node_storage)
#print([file_id])
file = node_storage_ro.get_real_name(file_id)
#print('ok2')
if file is not None:
file_names[file] = node_storage_ro.get_tags_of_file(file)
# dame el nombre del archivo y guardalo en file_names por denis
return file_names
def notify(self, other: RemoteObj):
"""
<other> might be our predecessor
"""
p = self.predecessor()
if p is None:
self.set_predecessor(other)
other_ro = get_remote(other)
other_storage = other_ro.get_remote_storage()
# send all my keys between self.id and other.id to other
self.storage.release_tags(self.id, other_storage)
self.storage.release_file_tags(self.id, other_storage)
self.storage.release_files(self.id, other_storage)
elif inrange(other.id, p.id, self.id):
self.set_predecessor(other)
other_ro = get_remote(other)
other_storage = other_ro.get_remote_storage()
# send all my keys between p.id and other.id to other
self.storage.release_tags(p.id, other_storage)
self.storage.release_file_tags(p.id, other_storage)
self.storage.release_files(p.id, other_storage)
elif not self.ping(p):
self.set_predecessor(other)
# load in storage all key from backups between other.id p.id
self.storage.load_from_tags_backup(other.id, p.id)
self.storage.load_from_file_tags_backup(other.id, p.id)
self.storage.load_from_files_backup(other.id, p.id)
def add(self, file_path_list: list, tag_list: list, ip: str, port: int):
file_list = [x.split('/')[-1] for x in file_path_list]
file_id_list = [hash_str(x) for x in file_list]
tag_id_list = [hash_str(x) for x in tag_list]
for file_path, file, file_id in zip(file_path_list, file_list,
file_id_list):
node = self.dht.find_successor(file_id)
node_ro = get_remote(node)
node_storage = node_ro.get_remote_storage()
node_storage_ro = get_remote(node_storage)
#print([file], [ip], [port])
#print(node_storage_ro)
node_storage_ro.upload(file, file_path, ip, port) # upload file
node_storage_ro.add_file_to_tags(file, tag_list)
#print('ok')
for tag, tag_id in zip(tag_list, tag_id_list):
node = self.dht.find_successor(tag_id)
node_ro = get_remote(node)
node_storage = node_ro.get_remote_storage()
node_storage_ro = get_remote(node_storage)
node_storage_ro.add_tag_to_files(tag_id, file_id_list)
def download(self, file_name):
file_id = hash_str(file_name)
node = self.dht.find_successor(file_id)
node_ro = get_remote(node)
node_storage = node_ro.get_remote_storage()
node_storage_ro = get_remote(node_storage)
data = node_storage_ro.download(file_id)
for d in data:
yield d
def join(self, remote_obj=None) -> True:
self._finger = [None] * M
self._predecessor = None
self.successor_list = []
if remote_obj is None:
try:
ns = Pyro4.locateNS()
for server in ns.list():
if "NameServer" not in server:
try:
with Pyro4.Proxy(ns.list()[server]) as p:
remote_info = p.get_info()
remote_obj = (remote_info.ip, remote_info.port)
break
except:
pass
except Exception as x:
print(x)
if remote_obj is None:
self.set_successor(self.get_info())
return True
#print(f'Trying connect to {remote_obj}')
other = remote_obj if isinstance(remote_obj, RemoteObj) else RemoteObj(
-1, *remote_obj)
if self.ping(other):
self.set_successor(get_remote(other).find_successor(self.id))
return True
return False
def _find_successor(self, id: int) -> RemoteObj:
"""
Ask node to find id’s successor
"""
p = self.find_predecessor(id)
ro = get_remote(p)
return ro.successor()
def send_files_backup_copy_to(self, remote_storage):
try:
remote_storage_ro = get_remote(remote_storage)
#for file in self.files_backups.keys():
for file in self.files.keys():
remote_storage_ro.upload_to_backup(self.files[file], self.ip, self.port)
except Exception as ex:
print(ex)
def stabilize(self):
"""
Periodically verify inmediate successor,
and tell the successor about this node
"""
if not self.ping(self.successor_list[0]):
return
s = self.successor()
s_ro = get_remote(s)
x = s_ro.predecessor()
if x is not None and inrange(x.id, self.id, s.id):
self.set_successor(x)
s = self.successor()
get_remote(s).notify(self.get_info())
def release_tags(self, p_id: int, remote_storage):
release_keys = [key for key in self.tags.keys() if inrange(key, p_id, remote_storage.id, (False, True))]
print(f'Releasing tags {release_keys} to {remote_storage.id}')
try:
remote_storage_ro = get_remote(remote_storage)
for tag in release_keys:
file_list = self.tags.pop(tag)
remote_storage_ro.add_tag_to_files(tag, file_list)
except Exception as ex:
print(ex)
def release_file_tags(self, p_id: int, remote_storage):
release_keys = [key for key in self.file_tags.keys() if inrange(hash_str(key), p_id, remote_storage.id, (False, True))]
print(f'Releasing files {release_keys} to {remote_storage.id}')
try:
remote_storage_ro = get_remote(remote_storage)
for file in release_keys:
tag_list = self.file_tags.pop(file)
remote_storage_ro.add_file_to_tags(file, tag_list)
except Exception as ex:
print(ex)
def delete(self, tag_list):
tag_id_list = [hash_str(x) for x in tag_list]
file_list = None
for tag, tag_id in zip(tag_list, tag_id_list):
node = self.dht.find_successor(tag_id)
node_ro = get_remote(node)
node_storage = node_ro.get_remote_storage()
node_storage_ro = get_remote(node_storage)
files = node_storage_ro.get_files_of_tag(tag_id)
if file_list is None:
file_list = files
else:
file_list.intersection_update(files)
for file_id in file_list:
node = self.dht.find_successor(file_id)
node_ro = get_remote(node)
node_storage = node_ro.get_remote_storage()
node_storage_ro = get_remote(node_storage)
#print([file_id])
node_storage_ro.del_files([file_id]) # del files
#print('ok1')
for tag, tag_id in zip(tag_list, tag_id_list):
node = self.dht.find_successor(tag_id)
node_ro = get_remote(node)
node_storage = node_ro.get_remote_storage()
node_storage_ro = get_remote(node_storage)
node_storage_ro.remove_files_of_tag(tag_id, file_list)
def delete_tags(self, tag_list, tag_list2):
tag_id_list = [hash_str(x) for x in tag_list]
tag_id_list2 = [hash_str(x) for x in tag_list2]
file_list = None
for tag, tag_id in zip(tag_list, tag_id_list):
node = self.dht.find_successor(tag_id)
node_ro = get_remote(node)
node_storage = node_ro.get_remote_storage()
node_storage_ro = get_remote(node_storage)
files = node_storage_ro.get_files_of_tag(tag_id)
if file_list is None:
file_list = files
else:
file_list.intersection_update(files)
for file_id in file_list:
node = self.dht.find_successor(file_id)
node_ro = get_remote(node)
node_storage = node_ro.get_remote_storage()
node_storage_ro = get_remote(node_storage)
node_storage_ro.remove_tags_of_file(file_id, tag_list2)
for tag2, tag_id2 in zip(tag_list2, tag_id_list2):
node = self.dht.find_successor(tag_id2)
node_ro = get_remote(node)
node_storage = node_ro.get_remote_storage()
node_storage_ro = get_remote(node_storage)
node_storage_ro.remove_files_of_tag(tag_id2, file_list)
def release_files(self, p_id: int, remote_storage):
release_keys = [key for key in self.files.keys() if inrange(key, p_id, remote_storage.id, (False, True))]
print(f'Releasing files {release_keys} to {remote_storage.id}')
try:
remote_storage_ro = get_remote(remote_storage)
for file in release_keys:
realname = self.files.pop(file)
self.files_backups[file] = realname
remote_storage_ro.upload_to_files(realname, self.ip, self.port)
except Exception as ex:
print(ex)
def find_predecessor(self, id: int) -> RemoteObj:
"""
Ask node to find id’s predecessor
"""
# corner case
# if self.id() == id:
# return self.predecessor()
p = self.get_info()
p_ro = get_remote(p)
while not inrange(id, p.id, p_ro.successor().id, (False, True)):
np = p_ro.closest_preceding_finger(id)
# corner case
# if p.id() == np.id():
# break
p = np
p_ro = get_remote(p)
return p
def ping(self, other: RemoteObj) -> bool:
print(
f'ID: {self.id} SUCC: {None if self._finger[0] is None else self._finger[0].id} SUCC-LIST: {[None if x is None else x.id for x in self.successor_list]}'
)
if other is None:
return False
try:
# print(f'Try Ping: {self} to {other}')
#print()
ro = get_remote(other)
return ro.pong()
except Exception as ex:
print(f'Ping: {ex}')
return False
def send_files_copy_to(self, remote_storage):
try:
remote_storage_ro = get_remote(remote_storage)
remote_storage_ro.update_file_tags_backup(self.file_tags)
except Exception as ex:
print(ex)
def update_successors(self):
# for i, s in enumerate(self.successor_list):
# if self.ping(s):
# if i > 0:
# s_ro = get_remote(s)
# s_ro.notify(self.get_info())
# self.successor_list = [s]
# break
# else:
# for i in range(1, M):
# finger = self.get_finger(i)
# if self.ping(finger) and finger.id != self.id:
# finger_ro = get_remote(finger)
# succ = finger_ro.find_successor(self.id)
# succ_ro = get_remote(succ)
# succ_ro.notify(self.get_info())
# self.successor_list = [succ]
# break
# self.successor_list = [succ for succ in self.successor_list if self.ping(succ)]
# if rd.random() <= 1/6 or len(self.successor_list) == 0:
# if len(self.successor_list) == 0:
# succ = self.successor()
# succ_ro = get_remote(succ)
# n_succ = succ_ro.find_successor(self.id) if succ.id != self.id else self._find_successor(self.id)
# self.successor_list = [n_succ]
# if len(self.successor_list) < M:
# l_succ = self.successor_list[-1]
# l_succ_ro = get_remote(l_succ)
# nl_succ = l_succ_ro.successor()
# if nl_succ.id != self.id and l_succ.id != nl_succ.id:
# self.successor_list.append(nl_succ)
# nl_succ_ro = get_remote(nl_succ)
# # sending replication
# nl_succ_storage = nl_succ_ro.get_remote_storage()
#
# succ = self.successor()
# self.set_successor(succ)
# succ_ro = get_remote(succ)
# succ_ro.notify(self.get_info())
succ = self.successor()
if succ.id != self.id:
successors = [succ]
suc_ro = get_remote(succ)
suc_list = suc_ro.get_successor_list()[:M - 1]
if suc_list and len(suc_list):
successors += suc_list
self.successor_list = successors
# replicating
copys = set(successors[:3])
for copy in copys:
copy_ro = get_remote(copy)
copy_storage = copy_ro.get_remote_storage()
self.storage.send_tags_copy_to(copy_storage)
self.storage.send_files_copy_to(copy_storage)
self.storage.send_files_backup_copy_to(copy_storage)
self.set_successor(succ)
