class DHT(object):
def __init__(self, local_address, remote_address=None):
self.local_ = Local(local_address, remote_address)
def set_wrap(msg):
return self._set(msg)
def get_wrap(msg):
return self._get(msg)
def backup_wrap(msg):
return self._backup(msg)
def dump_wrap(msg):
return self.dump()
def dumpall_wrap(msg):
return self.dumpall()
self.data_ = {}
self.backup_ = {}
self.shutdown_ = False
self.backupFlag = False
self.local_.register_command("set", set_wrap)
self.local_.register_command("get", get_wrap)
self.local_.register_command("backup", backup_wrap)
self.local_.register_command("dump", dump_wrap)
self.local_.register_command("dumpall", dumpall_wrap)
self.local_.stabilize = self.stabilize
self.local_.update_successors = self.update_successors
self.daemons_ = {}
self.daemons_['distribute_data'] = Daemon(self, 'distribute_data')
self.daemons_['distribute_data'].start()
self.daemons_['sendbackup'] = Daemon(self, 'sendbackup')
self.daemons_['sendbackup'].start()
self.local_.start()
@repeat_and_sleep(STABILIZE_INT)
@retry_on_socket_error(STABILIZE_RET)
def stabilize(self):
self.local_.log("stabilize")
suc = self.local_.successor()
# We may have found that x is our new successor iff
# - x = pred(suc(n))
# - x exists
# - x is in range (n, suc(n))
# - [n+1, suc(n)) is non-empty
# fix finger_[0] if successor failed
if suc.id() != self.local_.finger_[0].id():
self.local_.finger_[0] = suc
x = suc.predecessor()
if x != None and \
inrange(x.id(), self.local_.id(1), suc.id()) and \
self.local_.id(1) != suc.id() and \
x.ping():
self.local_.finger_[0] = x
# We notify our new successor about us
self.local_.successor().notify(self.local_)
# Keep calling us
for i in self.data_.iterkeys():
if self.backup_.has_key(i):
self.backup_.pop(i)
return True
@repeat_and_sleep(UPDATE_SUCCESSORS_INT)
@retry_on_socket_error(UPDATE_SUCCESSORS_RET)
def update_successors(self):
self.local_.log("update successor")
suc = self.local_.successor()
# if we are not alone in the ring, calculate
if suc.id() != self.local_.id():
successors = [suc]
suc_list = suc.get_successors()
if suc_list and len(suc_list):
successors += suc_list
# if everything worked, we update
if successors != self.local_.successors_:
self.local_.successors_ = successors
return True
def shutdown(self):
self.local_.shutdown()
self.shutdown_ = True
def _get(self, request):
try:
data = request
# we have the key
return json.dumps({'status': 'ok', 'data': self.get(data)})
except Exception:
# key not present
traceback.print_exc()
return json.dumps({'status': 'failed'})
def _set(self, request):
try:
key = request.split(' ')[0]
value = request.split(' ')[1]
self.set(key, value)
return json.dumps({'status': 'ok'})
# something is not working
except Exception:
traceback.print_exc()
return json.dumps({'status': 'failed'})
def get(self, key):
try:
print self.local_.id()
return self.data_[key] #['value']
except Exception:
# not in our range
suc = self.local_.find_successor(hash(key))
if self.local_.id() == suc.id():
# it's us but we don't have it
if self.backup_.has_key(key):
self.data_[key] = self.backup_[key]
self.backupFlag = True
print "Got at %s" % self.local_.id()
return self.data_[key] #['value']
return None
try:
response = suc.command('get %s' % key)
if not response:
raise Exception
value = json.loads(response)
if value['status'] != 'ok':
raise Exception
return value['data']
except Exception:
return None
@repeat_and_sleep(4)
def sendbackup(self):
if self.backupFlag == False:
pass
else:
temp = self.local_.successor()
if self.data_ != {}:
for i in [0, 1]:
if temp.id() == self.local_.id():
break
#while temp != self.local_:
response = temp.command(
'backup %s' %
json.dumps({
"id": i,
"address": self.local_.address(),
"data": self.data_
}))
temp = temp.successor()
print "Done backing up"
self.backupFlag = False
return True
def setactual(self, suc, key, value):
suc.command('set %s' % json.dumps({'key': key, 'value': value}))
def set(self, key, value):
# get successor for key
self.data_[key] = value #{"value":value, 'backedUp':False}
if key in self.backup_:
del self.backup_[key]
suc = self.local_.find_successor(hash(key))
if self.local_.id() == suc.id():
#its us
#print "Stored at %s" % self.local_.id()
self.backupFlag = True
try:
#self.data_[key]['backedUp']=True
return True
except Exception:
print traceback.print_exc(Exception)
return True
def _backup(self, request):
#print "Backing up: " + request + " at %s" % self.local_.id()
try:
data = json.loads(request)
id = data['id']
address = data['address']
value = data['data']
self.backup(id, address, value)
return json.dumps({'status': 'ok'})
# something is not working
except Exception:
traceback.print_exc()
return json.dumps({'status': 'failed'})
def backup(self, id, address, value):
#self.backup_ = {key: val for key, val in self.backup_.items()
# if val['address'] != address}
#self.backup_[id]={"address":address, "data":value}
for key in value:
self.backup_[key] = value[key]
return True
def printbackup(self):
temp = ""
for i in self.backup_:
temp += '{"' + i.__str__ + '":"' + str(self.backup_[i]) + '"}'
return temp
def dump(self):
dumpStr = "ID: %s" % self.local_.id() + "\n"\
+ "Data: %s" % json.dumps(self.data_) + "\n"\
+ "Backup: %s" % json.dumps(self.backup_) + "\n"
return dumpStr
def dumpall(self):
retStr = self.dump() + "\n\n"
temp = self.local_.successor()
temp2 = None
i = 0
while 1:
temp2 = self.local_
retStr += temp.command("dump") + "\n\n"
temp = temp.successor()
#print str(temp.id()) + ":" +str(temp2.id())
if temp.id() == temp2.id():
break
return retStr
@repeat_and_sleep(5)
def distribute_data(self):
to_remove = []
# to prevent from RTE in case data gets updated by other thread
keys = self.data_.keys()
for key in keys:
suc = self.local_.find_successor(hash(key))
if self.local_.id() != suc.id():
try:
suc.command("set " + key + " " + self.data_[key])
# print "moved %s into %s" % (key, node.id())
to_remove.append(key)
#print "migrated"
except socket.error:
print "error migrating"
# we'll migrate it next time
pass
# remove all the keys we do not own any more
if to_remove.__len__() != 0:
self.backupFlag = True
for key in to_remove:
if key not in self.backup_:
del self.data_[key]
# Keep calling us
return True
class Chat(object):
"""Data structure that represents a member of a chat network."""
def __init__(self, local_address, remote_address, name):
"""
:param local_address: address of the local node.
:param remote_address: address of the remote node to bootstrap from.
:param name: name belonging to the local node (name of the local user)
"""
self.local_ = Local(local_address, remote_address) # local chord node
self.local_.register_command("msg", self._accept_msg)
self.local_.register_command("named_msg", self._accept_named_msg)
self.local_.register_command("join", self._accept_join)
self.local_.register_command("leave", self._accept_leave)
self.local_.register_command("channel_msg", self._accept_channel_msg)
self.local_.register_command("migrate", self._accept_migrate)
self.local_.register_command("name", self._accept_name)
self.local_.register_command("delete_name", self._accept_delete_name)
self.local_.set_notify_handler(self._predecessor_change)
self.local_.start() # starting the local chord node
self.msg_handler_ = None # handler for outputting chat messages
self.mntnd_chnls = dict()
self.mntnd_names = dict()
self.joined_channels = set()
self.name_ = name
name_hash = hash_string(name)
print('registering our name')
node = self.local_.find_successor(name_hash)
node.user_command("name",
json.dumps({
'name': name,
'id': self.local_.id()
}))
self.local_.stabilize
def set_msg_handler(self, handler):
"""Sets handler of the incoming messages. The handler must be
a function accepting a message as its first parameter."""
self.msg_handler_ = handler
def _accept_name(self, msg):
"""Handler of a name command."""
try:
msg = json.loads(msg)
self.mntnd_names[msg['name']] = msg['id']
print('we map name ' + msg['name'] + ' to id ' + str(msg['id']))
except (KeyError, ValueError) as error:
print("bad message accepted", error)
return ''
def _accept_delete_name(self, msg):
"""Handler of a name command."""
try:
msg = json.loads(msg)
name = msg['name']
if name not in self.mntnd_names:
print("accepted delete command for unknown name " + name)
return ''
del self.mntnd_names[name]
print('deleted name ' + name + ' from maintained names')
except (KeyError, ValueError) as error:
print("bad message accepted", error)
return ''
def _accept_named_msg(self, msg):
"""Handler of a msg command."""
try:
msg = json.loads(msg)
name = msg['name']
if name == self.name_:
if self.msg_handler_ is not None:
self.msg_handler_(msg['msg'])
return ''
if not name in self.mntnd_names:
print('received a message for unknown name ' + name)
return ''
recp_id = self.mntnd_names[name]
print('received a message for name ' + name + ', sending it to ' +
str(recp_id))
node = self.local_.find_successor(recp_id)
node.user_command(
"msg", json.dumps({
'key': str(recp_id),
'value': msg['msg']
}))
except (KeyError, ValueError) as error:
print("bad message accepted", error)
return ''
def send_named_msg(self, recp, msg):
"""Handler of a named msg command."""
try:
name_hash = hash_string(recp)
print('sending message to ' + recp + ' with hash ' +
str(name_hash))
name_maintainer = self.local_.find_successor(name_hash)
name_maintainer.user_command(
'named_msg',
json.dumps({
'name': recp,
'msg': 'message from ' + self.name_ + ": " + msg
}))
print('message sent')
except socket.error:
print("send message failed")
def _accept_msg(self, msg):
"""Handler of a msg command."""
try:
msg = json.loads(msg)
if int(msg['key']) == self.local_.id() \
and self.msg_handler_ is not None:
self.msg_handler_(msg['value'])
else:
print('received message not for us but for: ', msg['key'])
except (KeyError, ValueError) as error:
print("bad message accepted", error)
return ''
def _accept_migrate(self, msg):
"""Handler of a migrate command."""
try:
msg = json.loads(msg)
print("migrate accepted")
migrate_channels = msg['channels']
for ch, ids in migrate_channels.items():
for id in ids:
self._add_node_to_channel(ch, id)
print('migrated channel ' + ch + ' with ' + str(len(ids)) +
' users.')
migrate_names = msg['names']
for name, id in migrate_channels.items():
mntnd_names[name] = id
print("migrate done")
except (KeyError, ValueError) as error:
print("bad migrate accepted", error)
return ''
def _add_node_to_channel(self, ch_name, id):
if not (ch_name in self.mntnd_chnls):
print('Creating new channel: ' + ch_name)
self.mntnd_chnls[ch_name] = set()
self.mntnd_chnls[ch_name].add(id)
print('Node with id: ' + id + ' joined channel: ' + ch_name)
def _accept_join(self, msg):
"""Handler of a join command."""
try:
msg = json.loads(msg)
self._add_node_to_channel(msg['channel_name'], msg['joining_id'])
except (KeyError, ValueError) as error:
print("bad join request accepted", error)
return ''
def _accept_leave(self, msg):
"""Handler of a leave command."""
try:
msg = json.loads(msg)
ch_name = msg['channel_name']
if not (ch_name in self.mntnd_chnls):
print('This node is not maintainer of channel ' + ch_name +
", but leave command was received.")
return ''
id = msg['leaving_id']
if not (id in self.mntnd_chnls[ch_name]):
print('User with id ' + id + ' is not member of channel ' +
ch_name + ".")
return ''
self.mntnd_chnls[ch_name].remove(id)
print('Node with id: ' + msg['leaving_id'] + ' left channel: ' +
ch_name)
except (KeyError, ValueError) as error:
print("bad message accepted", error)
return ''
def _accept_channel_msg(self, msg):
"""Handler of a channel_msg command."""
try:
msg = json.loads(msg)
ch_name = msg['channel_name']
if not (ch_name in self.mntnd_chnls):
print('This node is not maintainer of channel ' + ch_name +
", but send message command was received.")
return ''
for id in self.mntnd_chnls[ch_name]:
try:
node = self.local_.find_successor(int(id))
node.user_command(
"msg", json.dumps({
'key': id,
'value': msg['msg']
}))
except socket.error:
print("failed send of channel message to user " + id)
except (KeyError, ValueError) as error:
print("bad message accepted", error)
return ''
def send_msg(self, node_id, msg):
"""Attempts to send a message msg to a node with id node_id."""
try:
node_id = node_id % SIZE
str_hash = str(node_id)
print("trying to send message to: " + str(node_id) \
+ " with hash: " + str_hash)
node = self.local_.find_successor(node_id)
node.user_command(
"msg",
json.dumps({
'key': str_hash,
'value': 'message from ' + self.name_ + ": " + msg
}))
print("message sent")
except socket.error:
print("send failed")
def join_channel(self, ch_name):
""" joins channel with name ch_name """
try:
ch_hash = hash_string(ch_name)
print("trying to join channel " + str(ch_name) + " with hash: " +
str(ch_hash))
node = self.local_.find_successor(ch_hash)
node.user_command(
"join",
json.dumps({
'channel_name': ch_name,
'joining_id': str(self.local_.id())
}))
self.joined_channels.add(ch_name)
print("channel joined")
except socket.error:
print("channel join failed")
def leave_channel(self, ch_name):
""" leaves channel with name ch_name """
try:
ch_hash = hash_string(ch_name)
print("trying to leave channel " + str(ch_name) + " with hash: " +
str(ch_hash))
node = self.local_.find_successor(ch_hash)
node.user_command(
"leave",
json.dumps({
'channel_name': ch_name,
'leaving_id': str(self.local_.id())
}))
self.joined_channels.remove(ch_name)
print("channel left")
except socket.error:
print("channel leave failed")
def send_to_channel(self, ch_name, msg):
try:
ch_hash = hash_string(ch_name)
print("sending message to channel " + str(ch_name) +
" with hash: " + str(ch_hash))
node = self.local_.find_successor(ch_hash)
node.user_command(
"channel_msg",
json.dumps({
'channel_name':
ch_name,
'msg':
'message from ' + self.name_ + ' to channel ' + ch_name +
': ' + msg
}))
print("message to channel sent")
except socket.error:
print("channel send failed")
def _predecessor_change(self, new_predecessor):
"""Handler of the change of the predecessor event of the local chord
node.
:param new_predecessor: new predecessor of our node."""
if new_predecessor == self.local_:
return
migrate_chan = dict()
for ch, ids in list(self.mntnd_chnls.items()):
ch_id = hash_string(ch)
if not self.local_.is_ours(ch_id):
migrate_chan[ch] = list(ids)
del self.mntnd_chnls[ch]
print('migrating channel ' + ch + ' to a new predecessor.')
migrate_names = dict()
for name, id in list(self.mntnd_names.items()):
name_id = hash_string(name)
if not self.local_.is_ours(name_id):
migrate_names[name] = id
del self.mntnd_names[name]
print('migrating name ' + name + ' to a new predecessor.')
if len(migrate_chan) > 0 or len(migrate_names) > 0:
migrate = {'channels': migrate_chan, 'names': migrate_names}
new_predecessor.user_command('migrate', json.dumps(migrate))
def shutdown(self):
"""Shuts down the chat node."""
print("shutting down ...")
name_hash = hash_string(self.name_)
del_name_node = self.local_.find_successor(name_hash)
del_name_node.user_command("delete_name",
json.dumps({'name': self.name_}))
for ch in list(self.joined_channels):
self.leave_channel(ch)
migrate_chan = dict()
for ch, ids in list(self.mntnd_chnls.items()):
if len(ids) > 0:
migrate_chan[ch] = list(ids)
print('migrating channel ' + ch + ' to a new predecessor.')
if len(migrate_chan) > 0 or len(self.mntnd_names) > 0:
migrate = {'channels': migrate_chan, 'names': self.mntnd_names}
succ = self.local_.successor()
succ.user_command('migrate', json.dumps(migrate))
self.local_.shutdown()
class DHT(object):
def __init__(self, local_address, remote_address = None):
self.local_ = Local(local_address, remote_address)
def set_wrap(msg):
return self._set(msg)
def get_wrap(msg):
return self._get(msg)
def backup_wrap(msg):
return self._backup(msg)
def dump_wrap(msg):
return self.dump()
def dumpall_wrap(msg):
return self.dumpall()
self.data_ = {}
self.backup_ = {}
self.shutdown_ = False
self.backupFlag = False
self.local_.register_command("set", set_wrap)
self.local_.register_command("get", get_wrap)
self.local_.register_command("backup", backup_wrap)
self.local_.register_command("dump", dump_wrap)
self.local_.register_command("dumpall", dumpall_wrap)
self.local_.stabilize = self.stabilize
self.local_.update_successors = self.update_successors
self.daemons_ = {}
self.daemons_['distribute_data'] = Daemon(self, 'distribute_data')
self.daemons_['distribute_data'].start()
self.daemons_['sendbackup'] = Daemon(self, 'sendbackup')
self.daemons_['sendbackup'].start()
self.local_.start()
@repeat_and_sleep(STABILIZE_INT)
@retry_on_socket_error(STABILIZE_RET)
def stabilize(self):
self.local_.log("stabilize")
suc = self.local_.successor()
# We may have found that x is our new successor iff
# - x = pred(suc(n))
# - x exists
# - x is in range (n, suc(n))
# - [n+1, suc(n)) is non-empty
# fix finger_[0] if successor failed
if suc.id() != self.local_.finger_[0].id():
self.local_.finger_[0] = suc
x = suc.predecessor()
if x != None and \
inrange(x.id(), self.local_.id(1), suc.id()) and \
self.local_.id(1) != suc.id() and \
x.ping():
self.local_.finger_[0] = x
# We notify our new successor about us
self.local_.successor().notify(self.local_)
# Keep calling us
for i in self.data_.iterkeys():
if self.backup_.has_key(i):
self.backup_.pop(i)
return True
@repeat_and_sleep(UPDATE_SUCCESSORS_INT)
@retry_on_socket_error(UPDATE_SUCCESSORS_RET)
def update_successors(self):
self.local_.log("update successor")
suc = self.local_.successor()
# if we are not alone in the ring, calculate
if suc.id() != self.local_.id():
successors = [suc]
suc_list = suc.get_successors()
if suc_list and len(suc_list):
successors += suc_list
# if everything worked, we update
if successors!=self.local_.successors_:
self.local_.successors_ = successors
return True
def shutdown(self):
self.local_.shutdown()
self.shutdown_ = True
def _get(self, request):
try:
data = request
# we have the key
return json.dumps({'status':'ok', 'data':self.get(data)})
except Exception:
# key not present
traceback.print_exc()
return json.dumps({'status':'failed'})
def _set(self, request):
try:
key = request.split(' ')[0]
value = request.split(' ')[1]
self.set(key, value)
return json.dumps({'status':'ok'})
# something is not working
except Exception:
traceback.print_exc()
return json.dumps({'status':'failed'})
def get(self, key):
try:
print self.local_.id()
return self.data_[key]#['value']
except Exception:
# not in our range
suc = self.local_.find_successor(hash(key))
if self.local_.id() == suc.id():
# it's us but we don't have it
if self.backup_.has_key(key):
self.data_[key] = self.backup_[key]
self.backupFlag = True
print "Got at %s" % self.local_.id()
return self.data_[key]#['value']
return None
try:
response = suc.command('get %s' % key)
if not response:
raise Exception
value = json.loads(response)
if value['status'] != 'ok':
raise Exception
return value['data']
except Exception:
return None
@repeat_and_sleep(4)
def sendbackup(self):
if self.backupFlag==False:
pass
else:
temp = self.local_.successor()
if self.data_ != {}:
for i in [0,1]:
if temp.id() == self.local_.id():
break
#while temp != self.local_:
response = temp.command('backup %s' % json.dumps({"id":i,"address":self.local_.address(),
"data":self.data_}))
temp = temp.successor()
print "Done backing up"
self.backupFlag = False
return True
def setactual(self, suc, key, value):
suc.command('set %s' % json.dumps({'key':key,'value':value}))
def set(self, key, value):
# get successor for key
self.data_[key]=value#{"value":value, 'backedUp':False}
if key in self.backup_:
del self.backup_[key]
suc = self.local_.find_successor(hash(key))
if self.local_.id() == suc.id():
#its us
#print "Stored at %s" % self.local_.id()
self.backupFlag = True
try:
#self.data_[key]['backedUp']=True
return True
except Exception:
print traceback.print_exc(Exception)
return True
def _backup(self, request):
#print "Backing up: " + request + " at %s" % self.local_.id()
try:
data = json.loads(request)
id = data['id']
address = data['address']
value = data['data']
self.backup(id, address, value)
return json.dumps({'status':'ok'})
# something is not working
except Exception:
traceback.print_exc()
return json.dumps({'status':'failed'})
def backup(self, id, address, value):
#self.backup_ = {key: val for key, val in self.backup_.items()
# if val['address'] != address}
#self.backup_[id]={"address":address, "data":value}
for key in value:
self.backup_[key]=value[key]
return True
def printbackup(self):
temp = ""
for i in self.backup_:
temp += '{"'+i.__str__+'":"'+str(self.backup_[i])+'"}'
return temp
def dump(self):
dumpStr = "ID: %s" % self.local_.id() + "\n"\
+ "Data: %s" % json.dumps(self.data_) + "\n"\
+ "Backup: %s" % json.dumps(self.backup_) + "\n"
return dumpStr
def dumpall(self):
retStr = self.dump() + "\n\n"
temp = self.local_.successor()
temp2 = None
i = 0
while 1:
temp2 = self.local_
retStr+=temp.command("dump")+"\n\n"
temp = temp.successor()
#print str(temp.id()) + ":" +str(temp2.id())
if temp.id()==temp2.id():
break
return retStr
@repeat_and_sleep(5)
def distribute_data(self):
to_remove = []
# to prevent from RTE in case data gets updated by other thread
keys = self.data_.keys()
for key in keys:
suc = self.local_.find_successor(hash(key))
if self.local_.id() != suc.id():
try:
suc.command("set "+key+" "+self.data_[key])
# print "moved %s into %s" % (key, node.id())
to_remove.append(key)
#print "migrated"
except socket.error:
print "error migrating"
# we'll migrate it next time
pass
# remove all the keys we do not own any more
if to_remove.__len__()!=0:
self.backupFlag = True
for key in to_remove:
if key not in self.backup_:
del self.data_[key]
# Keep calling us
return True