def get_all(self, tipo):
# Returns all data store in every DHT node
return_data = {}
for key in self.data:
with open(self.data[key], 'r') as file:
data = file.read()
data = json.loads(data)
if not return_data.__contains__(
data['key']) and data['type'] == tipo:
return_data[key] = self.get(key)
succ = self.Node.successor()
while succ.id != self.Node.id:
if ping(succ):
with remote(succ, isDHT=True) as succDHT:
for key in succDHT.database:
data = succDHT.get(key)
if not return_data.__contains__(
data['key']) and data['type'] == tipo:
return_data[key] = succDHT.get(key)
with remote(succ) as next:
succ = next.successor()
else:
log("Lost Node {}".format(succ))
return return_data
def set(self, key, val):
# Check if its a write on a replica
with remote(self.Node.predecessor, isDHT=False) as pred:
with remote(self.Node.predecessor, isDHT=True) as predDHT:
if betweenclosedopen(key, pred.predecessor.id,
self.Node.predecessor.id):
predDHT.set(key, val)
else:
# Data will eventually be forwarded to the correct DHT peer if its not the local one
self.data[key] = "data/" + key
file = open(self.data[key], 'w')
file.write(json.dumps(val))
file.close()
def check_replicas(self):
if not self.Node.running:
return
replicated_data = self.rep_data
to_remove = []
for key in replicated_data.keys():
# Check if we must take over this replica in case the owner left the system
if self.Node.id == self.Node.successor().id or betweenclosedopen(
key, self.Node.predecessor.id, self.Node.id):
# We are now responsible for this data
self.data[key] = "data/" + replicated_data[key][4:]
shutil.copy(replicated_data[key], self.data[key])
log("Taking charge of key " + key)
to_remove.append(key)
# Check if we can clean up this replica
with remote(self.Node.predecessor, isDHT=False) as pred:
if not betweenclosedopen(key, pred.predecessor.id,
self.Node.predecessor.id):
# We no longer have to keep this replica
if not to_remove.__contains__(key):
to_remove.append(key)
for key in to_remove:
os.remove(self.rep_data[key])
del self.rep_data[key]
def get(self, key):
try:
with open(self.data[key], 'r') as file:
data = file.read()
return json.loads(data)
except Exception:
if betweenclosedopen(
key, self.Node.predecessor.id,
self.Node.id) or self.Node.successor().id == self.Node.id:
# We dont have the data yet
return None
succ = self.Node.find_successor(key) # *****
with remote(succ, isDHT=True) as succDHT: # *****
return succDHT.get(key) # *****
def replicate_data(self):
if not self.Node.running or self.Node.id == self.Node.successor().id:
# Do not replicate if the system has 1 Node or hasn't started
return
to_replicate = self.data
for key in to_replicate.keys():
if betweenclosedopen(key, self.Node.predecessor.id, self.Node.id):
with remote(
self.Node.successor(), isDHT=True
) as succ: # Push replicas to multiple successors ? 2 replicas enough ?
with open(to_replicate[key], 'r') as file:
x = file.read()
succ.take_replica(key, x)
def distribute_data(self):
if not self.Node.running or self.Node.id == self.Node.successor().id:
# No need to migrate data if the system has 1 Node or hasn't started
return
to_remove = []
database = self.data
for key in database.keys():
if not betweenclosedopen(key, self.Node.predecessor.id,
self.Node.id):
succ = self.Node.find_successor(key)
with remote(succ, isDHT=True) as succDHT:
with open(database[key], 'r') as file:
x = file.read()
succDHT.set(key, json.loads(x))
to_remove.append(key)
log("migrated key {} to node {}".format(key, succ.id))
# Remove migrated data
for key in to_remove:
os.remove(self.data[key])
del self.data[key]