def __init__(self, local_address, remote_address=None):
self.local_ = Local(local_address, remote_address)
def set_wrap(msg):
return self._set(msg)
def get_wrap(msg):
return self._get(msg)
def post_wrap(msg):
return self._post(msg)
def poll_wrap(msg):
return self._poll(msg)
self.data_ = {}
self.channels = {}
self.shutdown_ = False
self.local_.register_command("set", set_wrap)
self.local_.register_command("get", get_wrap)
self.local_.register_command("post", post_wrap)
self.local_.register_command("poll", poll_wrap)
self.daemons_ = {}
self.daemons_['distribute_data'] = Daemon(self, 'distribute_data')
self.daemons_['distribute_data'].start()
self.local_.start()
def __init__(self, local_address, remote_address = None):
self.local_ = Local(local_address, remote_address)
def set_wrap(msg):
return self._set(msg)
def get_wrap(msg):
return self._get(msg)
def backup_wrap(msg):
return self._backup(msg)
def dump_wrap(msg):
return self.dump()
def dumpall_wrap(msg):
return self.dumpall()
self.data_ = {}
self.backup_ = {}
self.shutdown_ = False
self.backupFlag = False
self.local_.register_command("set", set_wrap)
self.local_.register_command("get", get_wrap)
self.local_.register_command("backup", backup_wrap)
self.local_.register_command("dump", dump_wrap)
self.local_.register_command("dumpall", dumpall_wrap)
self.local_.stabilize = self.stabilize
self.local_.update_successors = self.update_successors
self.daemons_ = {}
self.daemons_['distribute_data'] = Daemon(self, 'distribute_data')
self.daemons_['distribute_data'].start()
self.daemons_['sendbackup'] = Daemon(self, 'sendbackup')
self.daemons_['sendbackup'].start()
self.local_.start()
def __init__(self, local_address, remote_address = None):
self.local_ = Local(local_address, remote_address)
def read_wrap(msg):
return self._read(msg)
def write_wrap(msg):
return self._write(msg)
def attr_wrap(msg):
return self._attr(msg)
self.data_ = {}
self.attr_ = {}
self.shutdown_ = False
self.local_.register_command("read", read_wrap)
self.local_.register_command("write", write_wrap)
self.local_.register_command("attr", attr_wrap)
self.local_.start()
def __init__(self, local_address, remote_address, name):
"""
:param local_address: address of the local node.
:param remote_address: address of the remote node to bootstrap from.
:param name: name belonging to the local node (name of the local user)
"""
self.local_ = Local(local_address, remote_address) # local chord node
self.local_.register_command("msg", self._accept_msg)
self.local_.register_command("named_msg", self._accept_named_msg)
self.local_.register_command("join", self._accept_join)
self.local_.register_command("leave", self._accept_leave)
self.local_.register_command("channel_msg", self._accept_channel_msg)
self.local_.register_command("migrate", self._accept_migrate)
self.local_.register_command("name", self._accept_name)
self.local_.register_command("delete_name", self._accept_delete_name)
self.local_.set_notify_handler(self._predecessor_change)
self.local_.start() # starting the local chord node
self.msg_handler_ = None # handler for outputting chat messages
self.mntnd_chnls = dict()
self.mntnd_names = dict()
self.joined_channels = set()
self.name_ = name
name_hash = hash_string(name)
print('registering our name')
node = self.local_.find_successor(name_hash)
node.user_command("name",
json.dumps({
'name': name,
'id': self.local_.id()
}))
self.local_.stabilize
def __init__(self, local_address, remote_address=None):
self.local_ = Local(local_address, remote_address)
def set_wrap(msg):
return self._set(msg)
def get_wrap(msg):
return self._get(msg)
def backup_wrap(msg):
return self._backup(msg)
def dump_wrap(msg):
return self.dump()
def dumpall_wrap(msg):
return self.dumpall()
self.data_ = {}
self.backup_ = {}
self.shutdown_ = False
self.backupFlag = False
self.local_.register_command("set", set_wrap)
self.local_.register_command("get", get_wrap)
self.local_.register_command("backup", backup_wrap)
self.local_.register_command("dump", dump_wrap)
self.local_.register_command("dumpall", dumpall_wrap)
self.local_.stabilize = self.stabilize
self.local_.update_successors = self.update_successors
self.daemons_ = {}
self.daemons_['distribute_data'] = Daemon(self, 'distribute_data')
self.daemons_['distribute_data'].start()
self.daemons_['sendbackup'] = Daemon(self, 'sendbackup')
self.daemons_['sendbackup'].start()
self.local_.start()
def __init__(self, local_address, remote_address = None):
self.local_ = Local(local_address, remote_address)
def set_wrap(msg):
return self._set(msg)
def get_wrap(msg):
return self._get(msg)
self.data_ = {}
self.shutdown_ = False
self.local_.register_command("set", set_wrap)
self.local_.register_command("get", get_wrap)
self.daemons_ = {}
self.daemons_['distribute_data'] = Daemon(self, 'distribute_data')
self.daemons_['distribute_data'].start()
self.local_.start()
class DFS(object):
def __init__(self, local_address, remote_address = None):
self.local_ = Local(local_address, remote_address)
def read_wrap(msg):
return self._read(msg)
def write_wrap(msg):
return self._write(msg)
def attr_wrap(msg):
return self._attr(msg)
self.data_ = {}
self.attr_ = {}
self.shutdown_ = False
self.local_.register_command("read", read_wrap)
self.local_.register_command("write", write_wrap)
self.local_.register_command("attr", attr_wrap)
self.local_.start()
def main():
usage = """
FUSEDFS: A filesystem implemented on top of a DHT.
""" + fuse.Fuse.fusage
if len(sys.argv) == 2:
local = Local(Address("127.0.0.1", sys.argv[1]))
else:
local = Local(Address("127.0.0.1", sys.argv[1]),
Address("127.0.0.1", sys.argv[2]))
local.start()
server = FUSEDFS(local,
version="%prog " + fuse.__version__,
usage=usage,
dash_s_do='setsingle')
server.parse(errex=1)
server.main()
class Chat(object):
"""Data structure that represents a member of a chat network."""
def __init__(self, local_address, remote_address, name):
"""
:param local_address: address of the local node.
:param remote_address: address of the remote node to bootstrap from.
:param name: name belonging to the local node (name of the local user)
"""
self.local_ = Local(local_address, remote_address) # local chord node
self.local_.register_command("msg", self._accept_msg)
self.local_.register_command("named_msg", self._accept_named_msg)
self.local_.register_command("join", self._accept_join)
self.local_.register_command("leave", self._accept_leave)
self.local_.register_command("channel_msg", self._accept_channel_msg)
self.local_.register_command("migrate", self._accept_migrate)
self.local_.register_command("name", self._accept_name)
self.local_.register_command("delete_name", self._accept_delete_name)
self.local_.set_notify_handler(self._predecessor_change)
self.local_.start() # starting the local chord node
self.msg_handler_ = None # handler for outputting chat messages
self.mntnd_chnls = dict()
self.mntnd_names = dict()
self.joined_channels = set()
self.name_ = name
name_hash = hash_string(name)
print('registering our name')
node = self.local_.find_successor(name_hash)
node.user_command("name",
json.dumps({
'name': name,
'id': self.local_.id()
}))
self.local_.stabilize
def set_msg_handler(self, handler):
"""Sets handler of the incoming messages. The handler must be
a function accepting a message as its first parameter."""
self.msg_handler_ = handler
def _accept_name(self, msg):
"""Handler of a name command."""
try:
msg = json.loads(msg)
self.mntnd_names[msg['name']] = msg['id']
print('we map name ' + msg['name'] + ' to id ' + str(msg['id']))
except (KeyError, ValueError) as error:
print("bad message accepted", error)
return ''
def _accept_delete_name(self, msg):
"""Handler of a name command."""
try:
msg = json.loads(msg)
name = msg['name']
if name not in self.mntnd_names:
print("accepted delete command for unknown name " + name)
return ''
del self.mntnd_names[name]
print('deleted name ' + name + ' from maintained names')
except (KeyError, ValueError) as error:
print("bad message accepted", error)
return ''
def _accept_named_msg(self, msg):
"""Handler of a msg command."""
try:
msg = json.loads(msg)
name = msg['name']
if name == self.name_:
if self.msg_handler_ is not None:
self.msg_handler_(msg['msg'])
return ''
if not name in self.mntnd_names:
print('received a message for unknown name ' + name)
return ''
recp_id = self.mntnd_names[name]
print('received a message for name ' + name + ', sending it to ' +
str(recp_id))
node = self.local_.find_successor(recp_id)
node.user_command(
"msg", json.dumps({
'key': str(recp_id),
'value': msg['msg']
}))
except (KeyError, ValueError) as error:
print("bad message accepted", error)
return ''
def send_named_msg(self, recp, msg):
"""Handler of a named msg command."""
try:
name_hash = hash_string(recp)
print('sending message to ' + recp + ' with hash ' +
str(name_hash))
name_maintainer = self.local_.find_successor(name_hash)
name_maintainer.user_command(
'named_msg',
json.dumps({
'name': recp,
'msg': 'message from ' + self.name_ + ": " + msg
}))
print('message sent')
except socket.error:
print("send message failed")
def _accept_msg(self, msg):
"""Handler of a msg command."""
try:
msg = json.loads(msg)
if int(msg['key']) == self.local_.id() \
and self.msg_handler_ is not None:
self.msg_handler_(msg['value'])
else:
print('received message not for us but for: ', msg['key'])
except (KeyError, ValueError) as error:
print("bad message accepted", error)
return ''
def _accept_migrate(self, msg):
"""Handler of a migrate command."""
try:
msg = json.loads(msg)
print("migrate accepted")
migrate_channels = msg['channels']
for ch, ids in migrate_channels.items():
for id in ids:
self._add_node_to_channel(ch, id)
print('migrated channel ' + ch + ' with ' + str(len(ids)) +
' users.')
migrate_names = msg['names']
for name, id in migrate_channels.items():
mntnd_names[name] = id
print("migrate done")
except (KeyError, ValueError) as error:
print("bad migrate accepted", error)
return ''
def _add_node_to_channel(self, ch_name, id):
if not (ch_name in self.mntnd_chnls):
print('Creating new channel: ' + ch_name)
self.mntnd_chnls[ch_name] = set()
self.mntnd_chnls[ch_name].add(id)
print('Node with id: ' + id + ' joined channel: ' + ch_name)
def _accept_join(self, msg):
"""Handler of a join command."""
try:
msg = json.loads(msg)
self._add_node_to_channel(msg['channel_name'], msg['joining_id'])
except (KeyError, ValueError) as error:
print("bad join request accepted", error)
return ''
def _accept_leave(self, msg):
"""Handler of a leave command."""
try:
msg = json.loads(msg)
ch_name = msg['channel_name']
if not (ch_name in self.mntnd_chnls):
print('This node is not maintainer of channel ' + ch_name +
", but leave command was received.")
return ''
id = msg['leaving_id']
if not (id in self.mntnd_chnls[ch_name]):
print('User with id ' + id + ' is not member of channel ' +
ch_name + ".")
return ''
self.mntnd_chnls[ch_name].remove(id)
print('Node with id: ' + msg['leaving_id'] + ' left channel: ' +
ch_name)
except (KeyError, ValueError) as error:
print("bad message accepted", error)
return ''
def _accept_channel_msg(self, msg):
"""Handler of a channel_msg command."""
try:
msg = json.loads(msg)
ch_name = msg['channel_name']
if not (ch_name in self.mntnd_chnls):
print('This node is not maintainer of channel ' + ch_name +
", but send message command was received.")
return ''
for id in self.mntnd_chnls[ch_name]:
try:
node = self.local_.find_successor(int(id))
node.user_command(
"msg", json.dumps({
'key': id,
'value': msg['msg']
}))
except socket.error:
print("failed send of channel message to user " + id)
except (KeyError, ValueError) as error:
print("bad message accepted", error)
return ''
def send_msg(self, node_id, msg):
"""Attempts to send a message msg to a node with id node_id."""
try:
node_id = node_id % SIZE
str_hash = str(node_id)
print("trying to send message to: " + str(node_id) \
+ " with hash: " + str_hash)
node = self.local_.find_successor(node_id)
node.user_command(
"msg",
json.dumps({
'key': str_hash,
'value': 'message from ' + self.name_ + ": " + msg
}))
print("message sent")
except socket.error:
print("send failed")
def join_channel(self, ch_name):
""" joins channel with name ch_name """
try:
ch_hash = hash_string(ch_name)
print("trying to join channel " + str(ch_name) + " with hash: " +
str(ch_hash))
node = self.local_.find_successor(ch_hash)
node.user_command(
"join",
json.dumps({
'channel_name': ch_name,
'joining_id': str(self.local_.id())
}))
self.joined_channels.add(ch_name)
print("channel joined")
except socket.error:
print("channel join failed")
def leave_channel(self, ch_name):
""" leaves channel with name ch_name """
try:
ch_hash = hash_string(ch_name)
print("trying to leave channel " + str(ch_name) + " with hash: " +
str(ch_hash))
node = self.local_.find_successor(ch_hash)
node.user_command(
"leave",
json.dumps({
'channel_name': ch_name,
'leaving_id': str(self.local_.id())
}))
self.joined_channels.remove(ch_name)
print("channel left")
except socket.error:
print("channel leave failed")
def send_to_channel(self, ch_name, msg):
try:
ch_hash = hash_string(ch_name)
print("sending message to channel " + str(ch_name) +
" with hash: " + str(ch_hash))
node = self.local_.find_successor(ch_hash)
node.user_command(
"channel_msg",
json.dumps({
'channel_name':
ch_name,
'msg':
'message from ' + self.name_ + ' to channel ' + ch_name +
': ' + msg
}))
print("message to channel sent")
except socket.error:
print("channel send failed")
def _predecessor_change(self, new_predecessor):
"""Handler of the change of the predecessor event of the local chord
node.
:param new_predecessor: new predecessor of our node."""
if new_predecessor == self.local_:
return
migrate_chan = dict()
for ch, ids in list(self.mntnd_chnls.items()):
ch_id = hash_string(ch)
if not self.local_.is_ours(ch_id):
migrate_chan[ch] = list(ids)
del self.mntnd_chnls[ch]
print('migrating channel ' + ch + ' to a new predecessor.')
migrate_names = dict()
for name, id in list(self.mntnd_names.items()):
name_id = hash_string(name)
if not self.local_.is_ours(name_id):
migrate_names[name] = id
del self.mntnd_names[name]
print('migrating name ' + name + ' to a new predecessor.')
if len(migrate_chan) > 0 or len(migrate_names) > 0:
migrate = {'channels': migrate_chan, 'names': migrate_names}
new_predecessor.user_command('migrate', json.dumps(migrate))
def shutdown(self):
"""Shuts down the chat node."""
print("shutting down ...")
name_hash = hash_string(self.name_)
del_name_node = self.local_.find_successor(name_hash)
del_name_node.user_command("delete_name",
json.dumps({'name': self.name_}))
for ch in list(self.joined_channels):
self.leave_channel(ch)
migrate_chan = dict()
for ch, ids in list(self.mntnd_chnls.items()):
if len(ids) > 0:
migrate_chan[ch] = list(ids)
print('migrating channel ' + ch + ' to a new predecessor.')
if len(migrate_chan) > 0 or len(self.mntnd_names) > 0:
migrate = {'channels': migrate_chan, 'names': self.mntnd_names}
succ = self.local_.successor()
succ.user_command('migrate', json.dumps(migrate))
self.local_.shutdown()
class DHT(object):
def __init__(self, local_address, remote_address=None):
self.local_ = Local(local_address, remote_address)
def set_wrap(msg):
return self._set(msg)
def get_wrap(msg):
return self._get(msg)
self.data_ = {}
self.shutdown_ = False
self.local_.register_command("set", set_wrap)
self.local_.register_command("get", get_wrap)
self.daemons_ = {}
self.daemons_['distribute_data'] = Daemon(self, 'distribute_data')
self.daemons_['distribute_data'].start()
self.local_.start()
def shutdown(self):
print("shutdown dht node")
self.local_.shutdown()
self.shutdown_ = True
def _get(self, request):
try:
data = json.loads(request)
# we have the key
return json.dumps({'status': 'ok', 'data': self.get(data['key'])})
except Exception:
# key not present
return json.dumps({'status': 'failed'})
def _set(self, request):
try:
data = json.loads(request)
key = data['key']
value = data['value']
self.set(key, value)
return json.dumps({'status': 'ok'})
except Exception:
# something is not working
return json.dumps({'status': 'failed'})
def get(self, key):
try:
return self.data_[key]
except Exception:
# not in our range
suc = self.local_.find_successor(
int(hashlib.sha256(str(key).encode("utf-8")).hexdigest(), 16))
if self.local_.id() == suc.id():
# it's us but we don't have it
return None
try:
response = suc.command('get %s' % json.dumps({'key': key}))
if not response:
raise Exception
value = json.loads(response)
if value['status'] != 'ok':
raise Exception
return value['data']
except Exception:
return None
def set(self, key, value):
# eventually it will distribute the keys
self.data_[key] = value
@repeat_and_sleep(5)
def distribute_data(self):
to_remove = []
# to prevent from RTE in case data gets updated by other thread
keys = list(self.data_.keys())
for key in keys:
if self.local_.predecessor() and \
not inrange(int(hashlib.sha256(str(key).encode("utf-8")).hexdigest(),16), self.local_.predecessor().id(1), self.local_.id(1)):
print(
int(
hashlib.sha256(str(key).encode("utf-8")).hexdigest(),
16))
try:
node = self.local_.find_successor(
int(
hashlib.sha256(
str(key).encode("utf-8")).hexdigest(), 16))
node.command("set %s" % json.dumps({
'key': key,
'value': self.data_[key]
}))
# print "moved %s into %s" % (key, node.id())
to_remove.append(key)
print("migrated")
except OSError:
print("error migrating")
# we'll migrate it next time
pass
# remove all the keys we do not own any more
for key in to_remove:
del self.data_[key]
# Keep calling us
return True
class DHT(object):
def __init__(self, local_address, remote_address=None):
self.local_ = Local(local_address, remote_address)
def set_wrap(msg):
return self._set(msg)
def get_wrap(msg):
return self._get(msg)
def backup_wrap(msg):
return self._backup(msg)
def dump_wrap(msg):
return self.dump()
def dumpall_wrap(msg):
return self.dumpall()
self.data_ = {}
self.backup_ = {}
self.shutdown_ = False
self.backupFlag = False
self.local_.register_command("set", set_wrap)
self.local_.register_command("get", get_wrap)
self.local_.register_command("backup", backup_wrap)
self.local_.register_command("dump", dump_wrap)
self.local_.register_command("dumpall", dumpall_wrap)
self.local_.stabilize = self.stabilize
self.local_.update_successors = self.update_successors
self.daemons_ = {}
self.daemons_['distribute_data'] = Daemon(self, 'distribute_data')
self.daemons_['distribute_data'].start()
self.daemons_['sendbackup'] = Daemon(self, 'sendbackup')
self.daemons_['sendbackup'].start()
self.local_.start()
@repeat_and_sleep(STABILIZE_INT)
@retry_on_socket_error(STABILIZE_RET)
def stabilize(self):
self.local_.log("stabilize")
suc = self.local_.successor()
# We may have found that x is our new successor iff
# - x = pred(suc(n))
# - x exists
# - x is in range (n, suc(n))
# - [n+1, suc(n)) is non-empty
# fix finger_[0] if successor failed
if suc.id() != self.local_.finger_[0].id():
self.local_.finger_[0] = suc
x = suc.predecessor()
if x != None and \
inrange(x.id(), self.local_.id(1), suc.id()) and \
self.local_.id(1) != suc.id() and \
x.ping():
self.local_.finger_[0] = x
# We notify our new successor about us
self.local_.successor().notify(self.local_)
# Keep calling us
for i in self.data_.iterkeys():
if self.backup_.has_key(i):
self.backup_.pop(i)
return True
@repeat_and_sleep(UPDATE_SUCCESSORS_INT)
@retry_on_socket_error(UPDATE_SUCCESSORS_RET)
def update_successors(self):
self.local_.log("update successor")
suc = self.local_.successor()
# if we are not alone in the ring, calculate
if suc.id() != self.local_.id():
successors = [suc]
suc_list = suc.get_successors()
if suc_list and len(suc_list):
successors += suc_list
# if everything worked, we update
if successors != self.local_.successors_:
self.local_.successors_ = successors
return True
def shutdown(self):
self.local_.shutdown()
self.shutdown_ = True
def _get(self, request):
try:
data = request
# we have the key
return json.dumps({'status': 'ok', 'data': self.get(data)})
except Exception:
# key not present
traceback.print_exc()
return json.dumps({'status': 'failed'})
def _set(self, request):
try:
key = request.split(' ')[0]
value = request.split(' ')[1]
self.set(key, value)
return json.dumps({'status': 'ok'})
# something is not working
except Exception:
traceback.print_exc()
return json.dumps({'status': 'failed'})
def get(self, key):
try:
print self.local_.id()
return self.data_[key] #['value']
except Exception:
# not in our range
suc = self.local_.find_successor(hash(key))
if self.local_.id() == suc.id():
# it's us but we don't have it
if self.backup_.has_key(key):
self.data_[key] = self.backup_[key]
self.backupFlag = True
print "Got at %s" % self.local_.id()
return self.data_[key] #['value']
return None
try:
response = suc.command('get %s' % key)
if not response:
raise Exception
value = json.loads(response)
if value['status'] != 'ok':
raise Exception
return value['data']
except Exception:
return None
@repeat_and_sleep(4)
def sendbackup(self):
if self.backupFlag == False:
pass
else:
temp = self.local_.successor()
if self.data_ != {}:
for i in [0, 1]:
if temp.id() == self.local_.id():
break
#while temp != self.local_:
response = temp.command(
'backup %s' %
json.dumps({
"id": i,
"address": self.local_.address(),
"data": self.data_
}))
temp = temp.successor()
print "Done backing up"
self.backupFlag = False
return True
def setactual(self, suc, key, value):
suc.command('set %s' % json.dumps({'key': key, 'value': value}))
def set(self, key, value):
# get successor for key
self.data_[key] = value #{"value":value, 'backedUp':False}
if key in self.backup_:
del self.backup_[key]
suc = self.local_.find_successor(hash(key))
if self.local_.id() == suc.id():
#its us
#print "Stored at %s" % self.local_.id()
self.backupFlag = True
try:
#self.data_[key]['backedUp']=True
return True
except Exception:
print traceback.print_exc(Exception)
return True
def _backup(self, request):
#print "Backing up: " + request + " at %s" % self.local_.id()
try:
data = json.loads(request)
id = data['id']
address = data['address']
value = data['data']
self.backup(id, address, value)
return json.dumps({'status': 'ok'})
# something is not working
except Exception:
traceback.print_exc()
return json.dumps({'status': 'failed'})
def backup(self, id, address, value):
#self.backup_ = {key: val for key, val in self.backup_.items()
# if val['address'] != address}
#self.backup_[id]={"address":address, "data":value}
for key in value:
self.backup_[key] = value[key]
return True
def printbackup(self):
temp = ""
for i in self.backup_:
temp += '{"' + i.__str__ + '":"' + str(self.backup_[i]) + '"}'
return temp
def dump(self):
dumpStr = "ID: %s" % self.local_.id() + "\n"\
+ "Data: %s" % json.dumps(self.data_) + "\n"\
+ "Backup: %s" % json.dumps(self.backup_) + "\n"
return dumpStr
def dumpall(self):
retStr = self.dump() + "\n\n"
temp = self.local_.successor()
temp2 = None
i = 0
while 1:
temp2 = self.local_
retStr += temp.command("dump") + "\n\n"
temp = temp.successor()
#print str(temp.id()) + ":" +str(temp2.id())
if temp.id() == temp2.id():
break
return retStr
@repeat_and_sleep(5)
def distribute_data(self):
to_remove = []
# to prevent from RTE in case data gets updated by other thread
keys = self.data_.keys()
for key in keys:
suc = self.local_.find_successor(hash(key))
if self.local_.id() != suc.id():
try:
suc.command("set " + key + " " + self.data_[key])
# print "moved %s into %s" % (key, node.id())
to_remove.append(key)
#print "migrated"
except socket.error:
print "error migrating"
# we'll migrate it next time
pass
# remove all the keys we do not own any more
if to_remove.__len__() != 0:
self.backupFlag = True
for key in to_remove:
if key not in self.backup_:
del self.data_[key]
# Keep calling us
return True
class DFS(object):
def __init__(self, local_address, remote_address=None):
self.local_ = Local(local_address, remote_address)
def read_wrap(msg):
return self._read(msg)
def write_wrap(msg):
return self._write(msg)
def attr_wrap(msg):
return self._attr(msg)
self.data_ = {}
self.attr_ = {}
self.shutdown_ = False
self.local_.register_command("read", read_wrap)
self.local_.register_command("write", write_wrap)
self.local_.register_command("attr", attr_wrap)
self.local_.start()
# helper function to eliminate duplicated code
def get_offsets(self, offset, size):
block_offset = offset / BLOCK_SIZE
start = offset % BLOCK_SIZE
end = min(start + size, BLOCK_SIZE)
return (block_offset, start, end)
def get_id(self, file_name, offset):
block_offset, start, end = self.get_offsets(offset, 0)
return "%s:%s" % (file_name, block_offset)
def get_hash(self, file_name, offset):
return hash(self.get_id(file_name, offset)) % settings.SIZE
def get_remote(self, file_name, offset):
hs = self.get_hash(file_name, offset)
suc = self.local_.find_successor(hs)
return suc
def _read(self, request):
# request = {'file_name':'my_file.txt', 'offset':<#NUMBER#>, 'size': <#NUMBER#>}
# response = {'status':'failed'} |
# {'status':'failed','code':<#CODE ERROR#>} |
# {'status':'redirect'}
# {'status':'ok','data':<#DATA READ AS B64#>}
try:
data = json.loads(request)
if not self.local_.is_ours(
self.get_hash(data['file_name'], data['offset'])):
return json.dumps({'status': 'redirect'})
# otherwise continue
result = self.read(data['file_name'], data['offset'], data['size'])
if type(result) == type(-1):
return json.dumps({'status': 'failed', 'code': result})
result = base64.b64encode(result)
return json.dumpds({'status': 'ok', 'data': total_read})
except Exception:
return json.dumps({'status': 'failed'})
def _write(self, request):
# request = {'file_name':'my_file.txt', 'offset':<#NUMBER#>, 'data':<#B64 ENCODED DATA#>}
# response = {'status':'failed'} |
# {'status':'failed','code':<#CODE ERROR#>} |
# {'status':'redirect'}
# {'status':'ok','bytes':<#BYTES WROTE#>}
try:
data = json.loads(request)
if not self.local_.is_ours(
self.get_hash(data['file_name'], data['offset'])):
return json.dumps({'status': 'redirect'})
result = self.write(data['file_name'],
base64.b64decode(data['data']), offset)
if result < 0:
return json.dumps({'status': 'failed', 'code': result})
else:
return json.dumps({'status': 'ok', 'bytes': result})
except Exception:
return json.dumps({'status': 'failed'})
def _attr(self, request):
# request = {'file_name':'my_file.txt'[,'size':<#NEW VALUE#>|,'mode':<#NEW MODE#>]}
# response = {'status':'failed'} |
# {'status':'failed','code':<#CODE ERROR#>} |
# {'status':'redirect'}
try:
data = json.loads(request)
if not self.local_.is_ours(self.get_hash(data['file_name'], 0)):
return json.dumps({'status': 'redirect'})
except Exception:
return json.dumps({'status': 'failed'})
def trunc_(self, request):
pass
def read(self, path, size, offset):
attr = self.attr(file_name)
if offset > attr.size:
return ""
block_offset, start, end = self.get_offsets(offset, size)
block_id = self.get_id(file_name, offset)
result = self.data_[block_id][start:end]
def write(self, path, buf, offset):
pass
def attr(self, path):
pass
def trunc_(self, request):
pass
def read(self, path, size, offset):
attr = self.attr(file_name)
if offset > attr.size:
return ""
block_offset, start, end = self.get_offsets(offset, size)
block_id = self.get_id(file_name, offset)
result = self.data_[block_id][start:end]
def write(self, path, buf, offset):
pass
def attr(self, path):
pass
if __name__ == "__main__":
import sys
if len(sys.argv) == 2:
dfs = Local(Address("127.0.0.1", sys.argv[1]))
else:
dfs = Local(Address("127.0.0.1", sys.argv[1]),
Address("127.0.0.1", sys.argv[2]))
class DHT(object):
def __init__(self, local_address, remote_address = None):
self.local_ = Local(local_address, remote_address)
def set_wrap(msg):
return self._set(msg)
def get_wrap(msg):
return self._get(msg)
def backup_wrap(msg):
return self._backup(msg)
def dump_wrap(msg):
return self.dump()
def dumpall_wrap(msg):
return self.dumpall()
self.data_ = {}
self.backup_ = {}
self.shutdown_ = False
self.backupFlag = False
self.local_.register_command("set", set_wrap)
self.local_.register_command("get", get_wrap)
self.local_.register_command("backup", backup_wrap)
self.local_.register_command("dump", dump_wrap)
self.local_.register_command("dumpall", dumpall_wrap)
self.local_.stabilize = self.stabilize
self.local_.update_successors = self.update_successors
self.daemons_ = {}
self.daemons_['distribute_data'] = Daemon(self, 'distribute_data')
self.daemons_['distribute_data'].start()
self.daemons_['sendbackup'] = Daemon(self, 'sendbackup')
self.daemons_['sendbackup'].start()
self.local_.start()
@repeat_and_sleep(STABILIZE_INT)
@retry_on_socket_error(STABILIZE_RET)
def stabilize(self):
self.local_.log("stabilize")
suc = self.local_.successor()
# We may have found that x is our new successor iff
# - x = pred(suc(n))
# - x exists
# - x is in range (n, suc(n))
# - [n+1, suc(n)) is non-empty
# fix finger_[0] if successor failed
if suc.id() != self.local_.finger_[0].id():
self.local_.finger_[0] = suc
x = suc.predecessor()
if x != None and \
inrange(x.id(), self.local_.id(1), suc.id()) and \
self.local_.id(1) != suc.id() and \
x.ping():
self.local_.finger_[0] = x
# We notify our new successor about us
self.local_.successor().notify(self.local_)
# Keep calling us
for i in self.data_.iterkeys():
if self.backup_.has_key(i):
self.backup_.pop(i)
return True
@repeat_and_sleep(UPDATE_SUCCESSORS_INT)
@retry_on_socket_error(UPDATE_SUCCESSORS_RET)
def update_successors(self):
self.local_.log("update successor")
suc = self.local_.successor()
# if we are not alone in the ring, calculate
if suc.id() != self.local_.id():
successors = [suc]
suc_list = suc.get_successors()
if suc_list and len(suc_list):
successors += suc_list
# if everything worked, we update
if successors!=self.local_.successors_:
self.local_.successors_ = successors
return True
def shutdown(self):
self.local_.shutdown()
self.shutdown_ = True
def _get(self, request):
try:
data = request
# we have the key
return json.dumps({'status':'ok', 'data':self.get(data)})
except Exception:
# key not present
traceback.print_exc()
return json.dumps({'status':'failed'})
def _set(self, request):
try:
key = request.split(' ')[0]
value = request.split(' ')[1]
self.set(key, value)
return json.dumps({'status':'ok'})
# something is not working
except Exception:
traceback.print_exc()
return json.dumps({'status':'failed'})
def get(self, key):
try:
print self.local_.id()
return self.data_[key]#['value']
except Exception:
# not in our range
suc = self.local_.find_successor(hash(key))
if self.local_.id() == suc.id():
# it's us but we don't have it
if self.backup_.has_key(key):
self.data_[key] = self.backup_[key]
self.backupFlag = True
print "Got at %s" % self.local_.id()
return self.data_[key]#['value']
return None
try:
response = suc.command('get %s' % key)
if not response:
raise Exception
value = json.loads(response)
if value['status'] != 'ok':
raise Exception
return value['data']
except Exception:
return None
@repeat_and_sleep(4)
def sendbackup(self):
if self.backupFlag==False:
pass
else:
temp = self.local_.successor()
if self.data_ != {}:
for i in [0,1]:
if temp.id() == self.local_.id():
break
#while temp != self.local_:
response = temp.command('backup %s' % json.dumps({"id":i,"address":self.local_.address(),
"data":self.data_}))
temp = temp.successor()
print "Done backing up"
self.backupFlag = False
return True
def setactual(self, suc, key, value):
suc.command('set %s' % json.dumps({'key':key,'value':value}))
def set(self, key, value):
# get successor for key
self.data_[key]=value#{"value":value, 'backedUp':False}
if key in self.backup_:
del self.backup_[key]
suc = self.local_.find_successor(hash(key))
if self.local_.id() == suc.id():
#its us
#print "Stored at %s" % self.local_.id()
self.backupFlag = True
try:
#self.data_[key]['backedUp']=True
return True
except Exception:
print traceback.print_exc(Exception)
return True
def _backup(self, request):
#print "Backing up: " + request + " at %s" % self.local_.id()
try:
data = json.loads(request)
id = data['id']
address = data['address']
value = data['data']
self.backup(id, address, value)
return json.dumps({'status':'ok'})
# something is not working
except Exception:
traceback.print_exc()
return json.dumps({'status':'failed'})
def backup(self, id, address, value):
#self.backup_ = {key: val for key, val in self.backup_.items()
# if val['address'] != address}
#self.backup_[id]={"address":address, "data":value}
for key in value:
self.backup_[key]=value[key]
return True
def printbackup(self):
temp = ""
for i in self.backup_:
temp += '{"'+i.__str__+'":"'+str(self.backup_[i])+'"}'
return temp
def dump(self):
dumpStr = "ID: %s" % self.local_.id() + "\n"\
+ "Data: %s" % json.dumps(self.data_) + "\n"\
+ "Backup: %s" % json.dumps(self.backup_) + "\n"
return dumpStr
def dumpall(self):
retStr = self.dump() + "\n\n"
temp = self.local_.successor()
temp2 = None
i = 0
while 1:
temp2 = self.local_
retStr+=temp.command("dump")+"\n\n"
temp = temp.successor()
#print str(temp.id()) + ":" +str(temp2.id())
if temp.id()==temp2.id():
break
return retStr
@repeat_and_sleep(5)
def distribute_data(self):
to_remove = []
# to prevent from RTE in case data gets updated by other thread
keys = self.data_.keys()
for key in keys:
suc = self.local_.find_successor(hash(key))
if self.local_.id() != suc.id():
try:
suc.command("set "+key+" "+self.data_[key])
# print "moved %s into %s" % (key, node.id())
to_remove.append(key)
#print "migrated"
except socket.error:
print "error migrating"
# we'll migrate it next time
pass
# remove all the keys we do not own any more
if to_remove.__len__()!=0:
self.backupFlag = True
for key in to_remove:
if key not in self.backup_:
del self.data_[key]
# Keep calling us
return True
class DHT(object):
def __init__(self, local_address, remote_address = None):
self.local_ = Local(local_address, remote_address)
def set_wrap(msg):
return self._set(msg)
def get_wrap(msg):
return self._get(msg)
self.data_ = {}
self.shutdown_ = False
self.local_.register_command("set", set_wrap)
self.local_.register_command("get", get_wrap)
self.daemons_ = {}
self.daemons_['distribute_data'] = Daemon(self, 'distribute_data')
self.daemons_['distribute_data'].start()
self.local_.start()
def shutdown(self):
self.local_.shutdown()
self.shutdown_ = True
def _get(self, request):
try:
data = json.loads(request)
# we have the key
return json.dumps({'status':'ok', 'data':self.get(data['key'])})
except Exception:
# key not present
return json.dumps({'status':'failed'})
def _set(self, request):
try:
data = json.loads(request)
key = data['key']
value = data['value']
self.set(key, value)
return json.dumps({'status':'ok'})
except Exception:
# something is not working
return json.dumps({'status':'failed'})
def get(self, key):
try:
return self.data_[key]
except Exception:
# not in our range
suc = self.local_.find_successor(hash(key))
if self.local_.id() == suc.id():
# it's us but we don't have it
return None
try:
response = suc.command('get %s' % json.dumps({'key':key}))
if not response:
raise Exception
value = json.loads(response)
if value['status'] != 'ok':
raise Exception
return value['data']
except Exception:
return None
def set(self, key, value):
# eventually it will distribute the keys
self.data_[key] = value
@repeat_and_sleep(5)
def distribute_data(self):
to_remove = []
# to prevent from RTE in case data gets updated by other thread
keys = self.data_.keys()
for key in keys:
if self.local_.predecessor() and \
not inrange(hash(key), self.local_.predecessor().id(1), self.local_.id(1)):
try:
node = self.local_.find_successor(hash(key))
node.command("set %s" % json.dumps({'key':key, 'value':self.data_[key]}))
# print "moved %s into %s" % (key, node.id())
to_remove.append(key)
print "migrated"
except socket.error:
print "error migrating"
# we'll migrate it next time
pass
# remove all the keys we do not own any more
for key in to_remove:
del self.data_[key]
# Keep calling us
return True
class DHT(object):
def __init__(self, local_address, remote_address=None):
self.local_ = Local(local_address, remote_address)
def set_wrap(msg):
return self._set(msg)
def get_wrap(msg):
return self._get(msg)
def post_wrap(msg):
return self._post(msg)
def poll_wrap(msg):
return self._poll(msg)
self.data_ = {}
self.channels = {}
self.shutdown_ = False
self.local_.register_command("set", set_wrap)
self.local_.register_command("get", get_wrap)
self.local_.register_command("post", post_wrap)
self.local_.register_command("poll", poll_wrap)
self.daemons_ = {}
self.daemons_['distribute_data'] = Daemon(self, 'distribute_data')
self.daemons_['distribute_data'].start()
self.local_.start()
def shutdown(self):
self.local_.shutdown()
self.shutdown_ = True
def _get(self, request):
try:
data = json.loads(request)
# we have the key
return json.dumps({'status': 'ok', 'data': self.get(data['key'])})
except Exception:
# key not present
return json.dumps({'status': 'failed'})
def _poll(self, request):
try:
data = json.loads(request)
key = data['key']
time = data['time']
# we have the key
#print(("polling",key,time))
return json.dumps({
'status': 'ok',
'data': self.pollChan(key, time)
})
except Exception:
# key not present
return json.dumps({'status': 'failed'})
def _post(self, request):
try:
data = json.loads(request)
key = data['key']
value = data['value']
# we have the key
self.postChan(key, value)
return json.dumps({'status': 'ok'})
except Exception:
# key not present
return json.dumps({'status': 'failed'})
def _set(self, request):
try:
data = json.loads(request)
key = data['key']
value = data['value']
self.set(key, value)
return json.dumps({'status': 'ok'})
except Exception:
# something is not working
return json.dumps({'status': 'failed'})
def pollChan(self, key, timestamp):
if key in self.channels.keys():
tmp = self.channels[key]
output = list(filter(tmp, lambda x: x[0] > timestamp))
return output
else:
return []
def postChan(self, key, msg):
now = time.time()
if key in list(self.channels.keys()):
self.channels[key].append((now, msg))
else:
self.channels[key] = [(now, msg)]
def get(self, key):
try:
return self.data_[key]
except Exception:
# not in our range
suc = self.local_.find_successor(hash(key))
if self.local_.id() == suc.id():
# it's us but we don't have it
return None
try:
response = suc.command('get %s' % json.dumps({'key': key}))
if not response:
raise Exception
value = json.loads(response)
if value['status'] != 'ok':
raise Exception
return value['data']
except Exception:
return None
def post(self, key, value):
# not in our range
suc = self.local_.find_successor(hash(key))
if self.local_.id() == suc.id():
# it's us but we don't have it
self.postChan(key, value)
return True
try:
response = suc.command('post %s' % json.dumps({
'key': key,
'value': value
}))
if not response:
raise Exception
value = json.loads(response)
if value['status'] != 'ok':
raise Exception
return True
except Exception:
return False
def poll(self, key, time):
#print("polling")
suc = self.local_.find_successor(hash(key))
if self.local_.id() == suc.id():
return self.pollChan(key, time)
else:
# not in our range
suc = self.local_.find_successor(hash(key))
if self.local_.id() == suc.id():
# it's us but we don't have it
return None
response = suc.command('poll %s' % json.dumps({
'key': key,
'time': time
}))
if not response:
raise Exception
value = json.loads(response)
if value['status'] != 'ok':
print(value)
return value['data']
"""
except Exception as e:
print(e)
return None
"""
def set(self, key, value):
# eventually it will distribute the keys
self.data_[key] = value
@repeat_and_sleep(5)
def distribute_data(self):
to_remove = []
# to prevent from RTE in case data gets updated by other thread
keys = list(self.data_.keys())
for key in keys:
if self.local_.predecessor() and \
not inrange(hash(key), self.local_.predecessor().id(1), self.local_.id(1)):
try:
node = self.local_.find_successor(hash(key))
node.command("set %s" % json.dumps({
'key': key,
'value': self.data_[key]
}))
# print "moved %s into %s" % (key, node.id())
to_remove.append(key)
print("migrated")
except socket.error:
print("error migrating")
# we'll migrate it next time
pass
# remove all the keys we do not own any more
for key in to_remove:
del self.data_[key]
# Keep calling us
return True