def client():
while True:
choice= int(input("What operation: 1. Upload 2. Download 3. Search 4. List File : "))
if choice==1:
channel = grpc.insecure_channel('127.0.0.1:3000')
fileName=input("FileName to be uploaded: ")
chunk_generator= get_file_chunks(fileName)
stub = fileService_pb2_grpc.FileserviceStub(channel)
response = stub.UploadFile(chunk_generator)
print(response)
elif choice==2:
channel = grpc.insecure_channel('127.0.0.1:3000')
name= input("Name of file to download")
stub = fileService_pb2_grpc.FileserviceStub(channel)
response = stub.DownloadFile(fileService_pb2.FileInfo(username="******", filename=name))
save_chunks_to_file(response, "downloads/own.pdf")
print("File downloaded. ")
elif choice==3:
channel = grpc.insecure_channel('127.0.0.1:3000')
name= input("Name of file to Search")
stub = fileService_pb2_grpc.FileserviceStub(channel)
response = stub.FileSearch(fileService_pb2.FileInfo(username="******", filename=name))
print(response)
elif choice==4:
channel = grpc.insecure_channel('127.0.0.1:3000')
name= input("Enter username: "******"akshay", filename=name))
print("File List. ", fileList)
def client():
while True:
choice= int(input("What operation: 1. Upload 2. Download 3. File Search 4. File List 5. File Delete : "))
if choice==1:
channel = grpc.insecure_channel('169.254.46.132:3000')
fileName=input("FileName to be uploaded: ")
# start = timer()
start_time = time.time()
chunk_generator= get_file_chunks(fileName)
# time.sleep(1)
# end = timer()
# print(end-start)
# print("Chunks generated in %s seconds" % (time.time() - start_time))
print("Chunks generated in %s seconds" % (str((time.time() - start_time))))
start_time = time.time()
stub = fileService_pb2_grpc.FileserviceStub(channel)
response = stub.UploadFile(chunk_generator)
print("File Uploaded in - %s seconds" % (str(round(time.time() - start_time,10))))
print(response)
elif choice==2:
channel = grpc.insecure_channel('169.254.46.132:3000')
name= input("Name of file to download")
start_time = time.time()
stub = fileService_pb2_grpc.FileserviceStub(channel)
res = stub.FileSearch(fileService_pb2.FileInfo(username="******", filename=name))
if res.success:
response = stub.DownloadFile(fileService_pb2.FileInfo(username="******", filename=name))
print("Files downloaded in %s seconds" % (str(round(time.time() - start_time,10))))
start_time = time.time()
save_chunks_to_file(response, "downloads/"+name)
print("Download chunks aggregated in %s seconds" % (str(round(time.time() - start_time,10))))
print("File downloaded. ")
else:
print("File not found")
elif choice==3:
channel = grpc.insecure_channel('169.254.46.132:3000')
name= input("Name of file to Search")
stub = fileService_pb2_grpc.FileserviceStub(channel)
response = stub.FileSearch(fileService_pb2.FileInfo(username="******", filename=name))
print(response)
elif choice==4:
channel = grpc.insecure_channel('169.254.46.132:3000')
name= input("Enter username: "******"akshay", filename=name))
print("File List. ", fileList)
elif choice==5:
channel = grpc.insecure_channel('169.254.46.132:3000')
name= input("Name of file to Delete: ")
stub = fileService_pb2_grpc.FileserviceStub(channel)
res = stub.FileSearch(fileService_pb2.FileInfo(username="******", filename=name))
if res.success:
response= stub.FileDelete(fileService_pb2.FileInfo(username="******", filename=name))
print(response)
else:
print("file not found")
def deleteDataAndMetaFromIndividualChunk(self, meta, username, filename):
node, seqNo, replicaNode = str(meta[0]), meta[1], str(meta[2])
metaDataKey = username + "_" + filename
dataChunkKey = username + "_" + filename + "_" + str(seqNo)
if (db.keyExists(metaDataKey) == 1):
print(
"deleteDataAndMetaFromIndividualChunk: Deleting the metadataEntry from local db :",
node)
db.deleteEntry(metaDataKey)
if (db.keyExists(dataChunkKey)):
print(
"deleteDataAndMetaFromIndividualChunk: Deleting the data chunk from local db:",
node)
db.deleteEntry(dataChunkKey)
active_ip_channel_dict = self.activeNodesChecker.getActiveChannels()
if (node != self.serverAddress and node in active_ip_channel_dict):
channel = active_ip_channel_dict[node]
stub = fileService_pb2_grpc.FileserviceStub(channel)
response = stub.FileDelete(
fileService_pb2.FileInfo(username=username,
filename=filename,
seqNo=seqNo))
if (response.success == True):
print(
"deleteDataAndMetaFromIndividualChunk : Successfully deleted chunk from node : ",
node)
else:
print(
"deleteDataAndMetaFromIndividualChunk : Chunk could not be deleted from node :",
node)
if (replicaNode != self.serverAddress and replicaNode
in self.activeNodesChecker.getActiveChannels()):
print("Deleting ")
channel = active_ip_channel_dict[replicaNode]
stub = fileService_pb2_grpc.FileserviceStub(channel)
response = stub.FileDelete(
fileService_pb2.FileInfo(username=username,
filename=filename,
seqNo=seqNo))
print(type(response.success))
if (response.success == True):
print("Successfully deleted chunk from ReplicaNode : ", node)
else:
print("Chunk could not be deleted from ReplicaNode :", node)
def FileDelete(self, request, context):
print("In FileDelete")
if (self.fileExists(request.username, request.filename) == False):
return fileService_pb2.ack(
success=False,
message="File {} does not exist.".format(request.filename))
fileMeta = db.parseMetaData(request.username, request.filename)
print("FileMeta = ", fileMeta)
primaryIP, replicaIP = -1, -1
channel1, channel2 = -1, -1
if (fileMeta[0] in self.clusterLeaders):
primaryIP = self.clusterLeaders[fileMeta[0]]
channel1 = self.clusterStatus.isChannelAlive(primaryIP)
if (fileMeta[1] in self.clusterLeaders):
replicaIP = self.clusterLeaders[fileMeta[1]]
channel2 = self.clusterStatus.isChannelAlive(replicaIP)
print("PrimarIP={}, replicaIP={}".format(primaryIP, replicaIP))
if (channel1 != -1):
print("Making fileDelete call to primary")
stub = fileService_pb2_grpc.FileserviceStub(channel1)
response = stub.FileDelete(
fileService_pb2.FileInfo(username=request.username,
filename=request.filename))
print("Received response = ", response.success)
if (channel2 != -1):
print("Making fileDelete call to replica")
stub = fileService_pb2_grpc.FileserviceStub(channel2)
response = stub.FileDelete(
fileService_pb2.FileInfo(username=request.username,
filename=request.filename))
print("Received response = ", response.success)
if (response.success == True):
db.deleteEntry(request.username + "_" + request.filename)
print("Successfully deleted.")
return fileService_pb2.ack(
success=True,
message="File successfully deleted from cluster : " +
fileMeta[0])
else:
print("Could not delete from replica")
return fileService_pb2.ack(success=False, message="Internal error")
def getDataFromIndividualNode(self, meta, username, filename):
print("Inside getDataFromIndividualNode")
print("Task Executed {}".format(threading.current_thread()))
node, seqNo, replicaNode = str(meta[0]), meta[1], str(meta[2])
data = bytes("",'utf-8')
result = {}
if(node==str(self.serverAddress)):
key = username + "_" + filename + "_" + str(seqNo)
data = db.getFileData(key)
else:
if(node in self.active_ip_channel_dict):
channel = self.active_ip_channel_dict[node]
print("Fetching Data from Node {}".format(node))
elif(replicaNode in self.active_ip_channel_dict):
channel = self.active_ip_channel_dict[replicaNode]
print("Fetching Data from Node {}".format(replicaNode))
else:
print("Both original and replica nodes are down!")
return
stub = fileService_pb2_grpc.FileserviceStub(channel)
responses = stub.DownloadFile(fileService_pb2.FileInfo(username = username, filename = filename, seqNo = seqNo))
for response in responses:
data+=response.data
self.seqDataMap[seqNo] = data
print("returning from the getDataFromIndividualNode")
def getDataFromNode(self, username, filename, chunk_id, destination):
channel = grpc.insecure_channel(destination)
channel2 = self.activeNodeObj.getActiveIpsDict()[destination]
stub = fileService_pb2_grpc.FileserviceStub(channel2)
response = stub.DownloadFile(
fileService_pb2.FileInfo(username=username,
filename=str(filename + chunk_id),
sequence_no=str(chunk_id)))
return response
def isFilePresent(stub):
userName = input("Enter Username: "******"Enter file name: ")
response = stub.FileSearch(fileService_pb2.FileInfo(username=userName, filename=fileName))
if(response.success==True):
print(response.message)
else:
print(response.message)
def DownloadFile(self, request, context):
# Check if file exists
if (self.fileExists(request.username, request.filename) == False):
return fileService_pb2.FileData(username=request.username,
filename=request.filename,
data=bytes("", 'utf-8'))
fileMeta = db.parseMetaData(request.username, request.filename)
primaryIP, replicaIP = -1, -1
channel1, channel2 = -1, -1
if (fileMeta[0] in self.clusterLeaders):
primaryIP = self.clusterLeaders[fileMeta[0]]
channel1 = self.clusterStatus.isChannelAlive(primaryIP)
if (fileMeta[1] in self.clusterLeaders):
replicaIP = self.clusterLeaders[fileMeta[1]]
channel2 = self.clusterStatus.isChannelAlive(replicaIP)
if (channel1):
stub = fileService_pb2_grpc.FileserviceStub(channel1)
responses = stub.DownloadFile(
fileService_pb2.FileInfo(username=request.username,
filename=request.filename))
for response in responses:
yield response
elif (channel2):
stub = fileService_pb2_grpc.FileserviceStub(channel2)
responses = stub.DownloadFile(
fileService_pb2.FileInfo(username=request.username,
filename=request.filename))
for response in responses:
yield response
else:
return fileService_pb2.FileData(username=request.username,
filename=request.filename,
data=bytes("", 'utf-8'))
def FileSearch(self, request, context):
if (self.fileExists(request.username, request.filename) == False):
return fileService_pb2.ack(
success=False,
message="File {} does not exist.".format(request.filename))
fileMeta = db.parseMetaData(request.username, request.filename)
primaryIP, replicaIP = -1, -1
channel1, channel2 = -1, -1
if (fileMeta[0] in self.clusterLeaders):
primaryIP = self.clusterLeaders[fileMeta[0]]
channel1 = self.clusterStatus.isChannelAlive(primaryIP)
if (fileMeta[1] in self.clusterLeaders):
replicaIP = self.clusterLeaders[fileMeta[1]]
channel2 = self.clusterStatus.isChannelAlive(replicaIP)
if (channel1 != -1):
stub = fileService_pb2_grpc.FileserviceStub(channel1)
response = stub.FileSearch(
fileService_pb2.FileInfo(username=request.username,
filename=request.filename))
if (channel2 != -1):
stub = fileService_pb2_grpc.FileserviceStub(channel2)
response = stub.FileSearch(
fileService_pb2.FileInfo(username=request.username,
filename=request.filename))
if (response.success == True):
return fileService_pb2.ack(success=True, message="File exists! ")
else:
return fileService_pb2.ack(
success=False, message="File does not exist in any cluster.")
def downloadTheFile(stub):
userName = input("Enter Username: "******"Enter file name: ")
data = bytes("",'utf-8')
sTime=time.time()
responses = stub.DownloadFile(fileService_pb2.FileInfo(username=userName, filename=fileName))
for response in responses:
fileName = response.filename
data += response.data
print("Time for Download = ", time.time()-sTime)
filePath=os.path.join('downloads', fileName)
saveFile = open(filePath, 'wb')
saveFile.write(data)
saveFile.close()
print("File Downloaded - ", fileName)
def UpdateFile(self, request_iterator, context):
username, filename = "", ""
fileData = bytes("", 'utf-8')
for request in request_iterator:
fileData += request.data
username, filename = request.username, request.filename
def getFileChunks(fileData):
# Maximum chunk size that can be sent
CHUNK_SIZE = 4000000
outfile = os.path.join('files', fileName)
sTime = time.time()
while True:
chunk = fileData.read(CHUNK_SIZE)
if not chunk: break
yield fileService_pb2.FileData(username=username,
filename=fileName,
data=chunk,
seqNo=1)
print("Time for upload= ", time.time() - sTime)
if (int(db.get("primaryStatus")) == 1):
channel = grpc.insecure_channel('{}'.format(self.serverAddress))
stub = fileService_pb2_grpc.FileserviceStub(channel)
response1 = stub.FileDelete(
fileService_pb2.FileInfo(username=userName, filename=fileName))
if (response1.success):
response2 = stub.UploadFile(getFileChunks(fileData))
if (response2.success):
return fileService_pb2.ack(
success=True, message="File suceessfully updated.")
else:
return fileService_pb2.ack(success=False,
message="Internal error.")
else:
return fileService_pb2.ack(success=False,
message="Internal error.")
def deleteTheFile(stub):
userName = input("Enter Username: "******"Enter file name: ")
response = stub.FileDelete(fileService_pb2.FileInfo(username=userName, filename=fileName))
print(response.message)