def setFingertable(self, node_list):
for i in node_list:
n = NodeID()
n.id = i.id
n.ip = i.ip
n.port = i.port
self.finger_table.append(n)
def _createNodeId(self, server_port):
# Create new object for NodeId
new_node_id = NodeID()
new_node_id.ip = str(socket.gethostbyname(
socket.gethostname())) # Set Node IP Address as string
new_node_id.port = server_port # Set Node Port Number as integer
# Set Node's ID as SHA256 hash value using "<Node_IP>:<Node_Port>" as string
encoded_address = (new_node_id.ip + COLON_STRING +
str(new_node_id.port)).encode(ENCODE_TYPE_UTF_8)
new_node_id.id = hashlib.sha256(encoded_address).hexdigest()
return new_node_id
def main():
transport = TSocket.TSocket(sys.argv[1], int(sys.argv[2]))
transport = TTransport.TBufferedTransport(transport)
protocol = TBinaryProtocol.TBinaryProtocol(transport)
client = FileStore.Client(protocol)
transport.open()
wFile = RFile()
wFileMeta = RFileMetadata()
wFileMeta.filename = "sample.txt"
sha256file = hashlib.sha256(wFileMeta.filename.encode())
wFileMeta.contentHash = sha256file.hexdigest()
wFile.meta = wFileMeta
wFile.content = "what now?"
#print('dddddddddddddddd : ',sha256file.hexdigest() )
client.writeFile(wFile)
rFile = RFile()
rFile = client.readFile('aaa.txt')
print('sha256: ', rFile.meta.contentHash)
print('content: ', rFile.content)
nodeID = NodeID()
#nodeID = client.getNodeSucc()
#keey = '69c9b6900f19eeddc1c9c99928410ddf045adb877f565487546c9cf05c9e2fd2'
#nodeID = client.findPred(sha256file.hexdigest())
#print('node', nodeID)
transport.close()
def __init__(self, ip, port):
self.files = {}
self.fingertable = []
self.nodeid_obj = NodeID()
self.nodeid_obj.id = unicode(
hashlib.sha256(str(ip) + ":" + str(port)).hexdigest())
self.nodeid_obj.ip = unicode(ip)
self.nodeid_obj.port = port
def __init__(self, port_no, ip_address):
self.finger_table = None
self.port_no = port_no
self.ip_address = ip_address
self.node = NodeID(self.calculate_hash(self.ip_address, self.port_no),
self.ip_address, self.port_no)
self.rFile = None
self.dictionary = {}
def findSucc(self, key):
qnode = NodeID()
qnode.id = self.sha256ID
qnode.ip = self.ip
qnode.port = self.port
nNode = NodeID()
nNode = self.findPred(key)
#print('predNode: ', nNode.id)
if qnode.id == key:
return qnode
else:
if nNode.id == qnode.id:
return self.getNodeSucc()
else:
ttransport = TSocket.TSocket(nNode.ip, nNode.port)
ttransport.open()
pprotocol = TBinaryProtocol.TBinaryProtocol(ttransport)
client = FileStore.Client(pprotocol)
cNode = NodeID()
cNode = client.getNodeSucc()
#ttransport.close()
return cNode
def __init__(self, host_addr, port_num, node_hash):
self.file_id = {}
self.node_list = []
self.port_num = port_num
self.node_hash = node_hash
self.host_addr = host_addr
self.is_file_owned = False
self.is_same_node = False
self.current_node = NodeID('', '', 0)
def findPred(self, key):
if not len(self.finger_table):
raise SystemException("Find Pred Error - Finger table is missing")
if self.NodeID >= self.finger_table[0].id:
if not (key <= self.NodeID and key > self.finger_table[0].id):
n = NodeID()
n.id = self.NodeID
n.ip = self.ip
n.port = int(self.port_num)
return n
else:
if key > self.NodeID and key <= self.finger_table[0].id:
n = NodeID()
n.id = self.NodeID
n.ip = self.ip
n.port = int(self.port_num)
return n
if self.NodeID >= key:
for i in reversed(self.finger_table):
if not (i.id <= self.NodeID and i.id > key):
t = TSocket.TSocket(i.ip, i.port)
t = TTransport.TBufferedTransport(t)
p = TBinaryProtocol.TBinaryProtocol(t)
c = FileStore.Client(p)
t.open()
k = c.findPred(key)
t.close()
return k
raise SystemException("Find Pred Error - Cannot find pred 1")
else:
for i in reversed(self.finger_table):
if i.id > self.NodeID and i.id <= key:
t = TSocket.TSocket(i.ip, i.port)
t = TTransport.TBufferedTransport(t)
p = TBinaryProtocol.TBinaryProtocol(t)
c = FileStore.Client(p)
t.open()
k = c.findPred(key)
t.close()
return k
raise SystemException("Find Pred Error - Cannot find pred 2")
def __init__(self, ip, port):
self.myFiles = {}
self.fingerTable = []
self.nodeID = NodeID()
self.nodeID.port = port
self.nodeID.ip = ip
self.id = int(
sha256((str(ip) + ':' + str(port)).encode()).hexdigest(), 16)
self.nodeID.id = sha256(
(str(ip) + ':' + str(port)).encode()).hexdigest()
print("{}:{}".format(ip, port))
def __init__(self):
self.fingerTable = []
self.contentHashes = []
self.myNode = NodeID()
port = int(sys.argv[1])
ipAddr = gethostbyname(gethostname())
idStr = gethostbyname(gethostname()) + ":" + sys.argv[1]
key = hashlib.sha256(idStr).hexdigest()
print(key)
self.myNode.ip = ipAddr
self.myNode.port = port
self.myNode.id = key
self.fileData = []
def __init__(self, port):
self.log = {}
self.serverNode = NodeID()
self.serverNode.ip = str(socket.gethostbyname(socket.gethostname()))
self.serverNode.id = hashlib.sha256(
str(socket.gethostbyname(socket.gethostname()) + ":" +
port).encode()).hexdigest()
self.serverNode.port = int(port)
print("Server Host Name is - ", socket.gethostname())
print("Server IP address (Host Address) is - ", self.serverNode.ip)
print("Server Port is - ", str(self.serverNode.port))
print("Server hash - ", self.serverNode.id)
self.chordSystemSize = pow(2, 256)
def __init__(self, port):
print("Intialized")
self.file_lookup = {}
self.finger_table = []
#self.hostname = socket.gethostname()
IP = socket.gethostbyname('localhost')
self.NodeID = NodeID()
self.NodeID.ip = IP
self.NodeID.port = int(port)
sha256 = hashlib.sha256()
sha256.update((IP + ':' + port).encode('utf-8'))
#id = hashlib.sha256(IP+':'+port)
#self.NodeID.id = id.hexdigest()
self.NodeID.id = sha256.hexdigest()
print(self.NodeID)
def findPred(self, key):
cNode = NodeID()
sha256id = hashlib.sha256(self.id.encode())
cNode.id = sha256id.hexdigest()
cNode.ip = self.ip
cNode.port = self.port
tempNode = NodeID()
if not self.fTable:
z = SystemException()
z.message = "no fTable"
raise z
tempNode = self.fTable[0]
x = self.checkInRange(key, cNode.id, tempNode.id)
#print('x:', x)
while not x:
for i in range(len(self.fTable) - 1, 0, -1):
temp = NodeID()
temp = self.fTable[i]
y = self.checkInRange(temp.id, cNode.id, key)
#print('temp.id: ', temp.id, '\n', 'cNode.id:', cNode.id, '\n', 'key:', key )
#print('y',y)
if y:
ttransport = TSocket.TSocket(temp.ip, temp.port)
#ttransport = TTransport.TBufferedTransport(ttransport)
ttransport.open()
pprotocol = TBinaryProtocol.TBinaryProtocol(ttransport)
client = FileStore.Client(pprotocol)
#ttransport.open()
sNode = NodeID()
sNode = client.findPred(key)
ttransport.close()
return sNode
break
return cNode
def readFile(self, fileC):
ipAddress = socket.gethostbyname(socket.gethostname())
file = ipAddress + ':' + str(port)
sha256key = hashlib.sha256(file.encode())
sHash = sha256key.hexdigest()
sha256val = hashlib.sha256(fileC.encode())
contentHash = sha256val.hexdigest()
checkNode = NodeID()
checkNode = self.findSucc(contentHash)
if self.sha256ID == checkNode.id:
if sHash in self.log:
#sha256val = hashlib.sha256(fileC.encode())
#sHashfile = sha256val.hexdigest()
sFile = RFile()
sFileMeta = RFileMetadata()
sFileMeta.filename = fileC
sFileMeta.version = self.version
sFileMeta.contentHash = contentHash
file = open(fileC, "r")
sFile.content = file.read()
print(sFile.content)
sFile.meta = sFileMeta
else:
x = SystemException()
x.message = "File Is not Present in server"
raise x
return sFile
else:
z = SystemException()
z.message = "server is not the file’s successor"
raise z
def writeFile(self, inputF):
#print('--------------------------------------------------')
ipAddress = socket.gethostbyname(socket.gethostname())
self.file = ipAddress + ':' + str(port)
sha256file = hashlib.sha256(self.file.encode())
self.sha256ID = sha256file.hexdigest()
#print ('IP and port ', file)
filename = inputF.meta.filename
contentHash = inputF.meta.contentHash
checkNode = NodeID()
checkNode = self.findSucc(contentHash)
#print('checknode id:', checkNode.id, 'self: ',self.sha256ID)
if self.sha256ID == checkNode.id:
if self.sha256ID in self.log:
#x = self.log[self.sha256ID].meta.version
#self.log[self.sha256ID].meta.version = x+1
#self.log[self.sha256ID].content = inputF.content
#x = RFile()
#x1 = RFileMetadata()
#x1 = self.log[self.sha256ID].meta
#x1.version = x1.version + 1
#self.log[self.sha256ID].meta = x1
self.version = self.version + 1
self.content = inputF.content
#x = self.log[self.sha256ID]
#x.meta = RFileMetadata()
#x.meta.version = x.meta.version + 1
#x1 = x.meta
#x1.version = x.meta.version + 1
#x.RFileMetadata = x1
#x.meta.version = x.meta.version+1
#x.content = inputF.content
#self.log[self.sha256ID] = x
#self.log[sha256ID].meta.version = x+1
#self.log[sha256ID].content = inputF.content
#x = serverFile.meta.version
#serverFile.meta.version = x+1
#serverFile.meta.contentHash = inputF.meta.contentHash
#serverFile.content = inputF.content
file = open(inputF.meta.filename, "w+")
file.write(inputF.content)
else:
serverFile = RFile()
sMeta = RFileMetadata()
file = open(filename, "w+")
file.write(inputF.content)
sMeta.filename = filename
sMeta.version = 0
self.version = sMeta.version
sMeta.contentHash = contentHash
serverFile.content = inputF.content
self.content = serverFile.content
serverFile.RFileMetadata = sMeta
self.log[self.sha256ID] = serverFile
file.close()
else:
y = SystemException()
y.message = "Rserver is not the file’s successor"
raise y
def node_instance(ip, port, id):
n = NodeID()
n.ip = ip
n.port = int(port)
n.id = id
return n
class Handler():
fingerTable = []
rfileArray = []
currentNode = NodeID()
files = []
host_name = socket.gethostname()
host = socket.gethostbyname(host_name)
port = sys.argv[1]
id1 = sha_256_id(host, port)
currentNode.id = id1
currentNode.ip = host
currentNode.port = port
print currentNode
def writeFile(self, rFile):
exists = False
node = self.findSucc(rFile.meta.contentHash)
if node != self.currentNode:
raise SystemException("WriteFile, file does not exist")
for x in self.files:
if x.get("filename" == rFile.meta.filename):
exists = True
if exists:
rFile.meta.version = rFile.meta.version + 1
file1 = open(rFile.meta.filename, "w")
file1.write(rFile.content)
else:
rFile.meta.version = 1
file1 = open(rFile.meta.filename, "w")
rFile.contentHash = sha_256(rFile.meta.filename)
file1.write(rFile.content)
self.files.append(rFile)
def readFile(self, filename):
key = sha_256(filename)
node = self.findSucc(key)
if node != self.currentNode:
raise SystemException("ReadFile, file does not exist")
exists = False
index = 0
for x in range(0, len(self.files)):
if self.files[x].meta.filename == filename:
exists = True
index = x
temp = self.files[index]
rFile1 = RFile()
if exists:
print("exist")
meta = RFileMetadata()
meta.filename = filename
meta.version = temp.meta.version
meta.contentHash = temp.meta.contentHash
rFile1.meta = meta
rFile1.content = temp.content
return rFile1
raise SystemException("ReadFile, file not found")
def setFingertable(self, node_list):
for i in node_list:
node = createNode(i.id, i.ip, i.port)
self.fingerTable.append(node)
def findSucc(self, key):
if len(self.fingerTable) == 0:
raise SystemException("no finger table")
n = self.findPred(key)
if not n.id == self.currentNode.id:
transport, client = createClient(n.ip, n.port)
s = client.getNodeSucc()
transport.close()
return s
else:
return self.getNodeSucc()
def getNodeSucc(self):
if len(self.fingerTable) == 0:
raise SystemException("no finger table")
return self.fingerTable[0]
def findPred(self, key):
if len(self.fingerTable) == 0:
raise SystemException("no finger table")
# ***************** check first finger table index *******************
if self.currentNode.id >= self.fingerTable[0].id:
if not (key <= self.currentNode.id
and key > self.fingerTable[0].id):
n = createNode(self.currentNode.id, self.currentNode.ip,
int(self.currentNode.port))
return n
else:
if key > self.currentNode.id and key <= self.fingerTable[0].id:
n = createNode(self.currentNode.id, self.currentNode.ip,
int(self.currentNode.port))
return n
# ***************** check other finger table index *******************
if self.currentNode.id >= key:
for i in range(len(self.fingerTable) - 1, 0, -1):
if not (self.fingerTable[i].id <= self.currentNode.id
and self.fingerTable[i].id > key):
t, c = createClient(self.fingerTable[i].ip,
self.fingerTable[i].port)
k = c.findPred(key)
t.close()
return k
raise SystemException("FindPred error")
else:
for i in range(len(self.fingerTable) - 1, 0, -1):
if self.fingerTable[
i].id > self.currentNode.id and self.fingerTable[
i].id <= key:
transport, client = createClient(self.fingerTable[i].ip,
self.fingerTable[i].port)
k = client.findPred(key)
transport.close()
return k
raise SystemException("FindPred error")
def createNode(id1, ip, port):
node = NodeID()
node.id = id1
node.ip = ip
node.port = port
return node
return self.ft[0]
# Destructor
def __del__(self):
# Remove directory
try:
shutil.rmtree(self.serverDir)
except:
print('Failed to remove %s' % self.serverDir)
print('%i: Node at %s:%i leaving' % (nodeID.port, nodeID.ip, nodeID.port))
if __name__ == '__main__':
port = int(sys.argv[1])
ip = socket.gethostbyname(socket.gethostname())
nodeID = NodeID()
nodeID.id = getID(ip, port)
nodeID.ip = ip
nodeID.port = port
handler = FileStoreHandler()
processor = FileStore.Processor(handler)
transport = TSocket.TServerSocket(port=port)
tfactory = TTransport.TBufferedTransportFactory()
pfactory = TBinaryProtocol.TBinaryProtocolFactory()
server = TServer.TSimpleServer(processor, transport, tfactory, pfactory)
# You could do one of these for a multithreaded server
# server = TServer.TThreadedServer(
# processor, transport, tfactory, pfactory)
def __init__(self):
self.log = {}
self.files = {}
self.finger_table = []
self.node_id = NodeID()
self.node_id.ip = socket.gethostbyname(socket.gethostname())
def addNode(self, host, port):
new_node_key = hashlib.sha256(host + ":" + str(port)).hexdigest()
new_node = NodeID()
new_node.ip = host
new_node.port = port
new_node.id = new_node_key
## Make its fingertable (current node to find all answers)
new_fingertable = []
for i in range(256):
finger_key = \
hex((int(new_node_key, 16) + (2**i)) % (2**256)).strip("0x").strip('L')
succ = self.findSucc(finger_key)
if(self.contains(new_node_key, finger_key, succ.id)):
new_fingertable.append(new_node)
else:
new_fingertable.append(self.findSucc(finger_key))
## Launch new server
Popen(["python2", "server.py"] + [str(port)])
sleep(3)
## Update other nodes fingertables
transport = TSocket.TSocket(host, port)
# Buffering is critical. Raw sockets are very slow
transport = TTransport.TBufferedTransport(transport)
# Wrap in a protocol
protocol = TBinaryProtocol.TBinaryProtocol(transport)
# Create a client to use the protocol encoder
client = FileStore.Client(protocol)
transport.open()
client.setFingertable(new_fingertable)
transport.close()
new_node_succ = self.findSucc(new_node_key)
new_node_pred = self.findPred(new_node_key)
succ_files = []
if(new_node_succ.id == self.myNode.id):
succ_files = self.getFiles()
else:
## Update other nodes fingertables
transport = TSocket.TSocket(new_node_succ.ip, new_node_succ.port)
# Buffering is critical. Raw sockets are very slow
transport = TTransport.TBufferedTransport(transport)
# Wrap in a protocol
protocol = TBinaryProtocol.TBinaryProtocol(transport)
# Create a client to use the protocol encoder
client = FileStore.Client(protocol)
transport.open()
succ_files = client.getFiles()
transport.close()
# New node must take ownership of some files from successor,
# while successor must relinquish ownership of some
for file_obj in succ_files:
if(self.contains(file_obj.meta.contentHash,\
new_node_pred.id,\
new_node_key)):
## File belongs to new node
if(new_node_succ.id == self.myNode.id):
self.removeFile(file_obj.meta.contentHash)
else:
## Update other nodes fingertables
transport = TSocket.TSocket(new_node_succ.ip, new_node_succ.port)
# Buffering is critical. Raw sockets are very slow
transport = TTransport.TBufferedTransport(transport)
# Wrap in a protocol
protocol = TBinaryProtocol.TBinaryProtocol(transport)
# Create a client to use the protocol encoder
client = FileStore.Client(protocol)
transport.open()
client.removeFile(file_obj.meta.contentHash)
## Update other nodes fingertables
transport = TSocket.TSocket(host, port)
# Buffering is critical. Raw sockets are very slow
transport = TTransport.TBufferedTransport(transport)
# Wrap in a protocol
protocol = TBinaryProtocol.TBinaryProtocol(transport)
# Create a client to use the protocol encoder
client = FileStore.Client(protocol)
transport.open()
client.addFile(file_obj)
def decode_sha256(s):
import hashlib
sha256 = hashlib.sha256()
sha256.update(s)
return sha256.hexdigest()
if __name__ == '__main__':
port = sys.argv[1]
import socket
ip = socket.gethostbyname(socket.getfqdn())
handler = FileStoreHandler(
NodeID(id=decode_sha256(ip + ":" + port), ip=ip, port=int(port, 10)))
processor = FileStore.Processor(handler)
transport = TSocket.TServerSocket(port=port)
tfactory = TTransport.TBufferedTransportFactory()
pfactory = TBinaryProtocol.TBinaryProtocolFactory()
server = TServer.TSimpleServer(processor, transport, tfactory, pfactory)
# # You could do one of these for a multithreaded server
# server = TServer.TThreadedServer(
# processor, transport, tfactory, pfactory)
# server = TServer.TThreadPoolServer(
# processor, transport, tfactory, pfactory)
print('Starting the server...')
server.serve()