def client(): while True: choice= int(input("What operation: 1. Upload 2. Download 3. Search 4. List File : ")) if choice==1: channel = grpc.insecure_channel('127.0.0.1:3000') fileName=input("FileName to be uploaded: ") chunk_generator= get_file_chunks(fileName) stub = fileService_pb2_grpc.FileserviceStub(channel) response = stub.UploadFile(chunk_generator) print(response) elif choice==2: channel = grpc.insecure_channel('127.0.0.1:3000') name= input("Name of file to download") stub = fileService_pb2_grpc.FileserviceStub(channel) response = stub.DownloadFile(fileService_pb2.FileInfo(username="******", filename=name)) save_chunks_to_file(response, "downloads/own.pdf") print("File downloaded. ") elif choice==3: channel = grpc.insecure_channel('127.0.0.1:3000') name= input("Name of file to Search") stub = fileService_pb2_grpc.FileserviceStub(channel) response = stub.FileSearch(fileService_pb2.FileInfo(username="******", filename=name)) print(response) elif choice==4: channel = grpc.insecure_channel('127.0.0.1:3000') name= input("Enter username: "******"akshay", filename=name)) print("File List. ", fileList)
def client(): while True: choice= int(input("What operation: 1. Upload 2. Download 3. File Search 4. File List 5. File Delete : ")) if choice==1: channel = grpc.insecure_channel('169.254.46.132:3000') fileName=input("FileName to be uploaded: ") # start = timer() start_time = time.time() chunk_generator= get_file_chunks(fileName) # time.sleep(1) # end = timer() # print(end-start) # print("Chunks generated in %s seconds" % (time.time() - start_time)) print("Chunks generated in %s seconds" % (str((time.time() - start_time)))) start_time = time.time() stub = fileService_pb2_grpc.FileserviceStub(channel) response = stub.UploadFile(chunk_generator) print("File Uploaded in - %s seconds" % (str(round(time.time() - start_time,10)))) print(response) elif choice==2: channel = grpc.insecure_channel('169.254.46.132:3000') name= input("Name of file to download") start_time = time.time() stub = fileService_pb2_grpc.FileserviceStub(channel) res = stub.FileSearch(fileService_pb2.FileInfo(username="******", filename=name)) if res.success: response = stub.DownloadFile(fileService_pb2.FileInfo(username="******", filename=name)) print("Files downloaded in %s seconds" % (str(round(time.time() - start_time,10)))) start_time = time.time() save_chunks_to_file(response, "downloads/"+name) print("Download chunks aggregated in %s seconds" % (str(round(time.time() - start_time,10)))) print("File downloaded. ") else: print("File not found") elif choice==3: channel = grpc.insecure_channel('169.254.46.132:3000') name= input("Name of file to Search") stub = fileService_pb2_grpc.FileserviceStub(channel) response = stub.FileSearch(fileService_pb2.FileInfo(username="******", filename=name)) print(response) elif choice==4: channel = grpc.insecure_channel('169.254.46.132:3000') name= input("Enter username: "******"akshay", filename=name)) print("File List. ", fileList) elif choice==5: channel = grpc.insecure_channel('169.254.46.132:3000') name= input("Name of file to Delete: ") stub = fileService_pb2_grpc.FileserviceStub(channel) res = stub.FileSearch(fileService_pb2.FileInfo(username="******", filename=name)) if res.success: response= stub.FileDelete(fileService_pb2.FileInfo(username="******", filename=name)) print(response) else: print("file not found")
def deleteDataAndMetaFromIndividualChunk(self, meta, username, filename): node, seqNo, replicaNode = str(meta[0]), meta[1], str(meta[2]) metaDataKey = username + "_" + filename dataChunkKey = username + "_" + filename + "_" + str(seqNo) if (db.keyExists(metaDataKey) == 1): print( "deleteDataAndMetaFromIndividualChunk: Deleting the metadataEntry from local db :", node) db.deleteEntry(metaDataKey) if (db.keyExists(dataChunkKey)): print( "deleteDataAndMetaFromIndividualChunk: Deleting the data chunk from local db:", node) db.deleteEntry(dataChunkKey) active_ip_channel_dict = self.activeNodesChecker.getActiveChannels() if (node != self.serverAddress and node in active_ip_channel_dict): channel = active_ip_channel_dict[node] stub = fileService_pb2_grpc.FileserviceStub(channel) response = stub.FileDelete( fileService_pb2.FileInfo(username=username, filename=filename, seqNo=seqNo)) if (response.success == True): print( "deleteDataAndMetaFromIndividualChunk : Successfully deleted chunk from node : ", node) else: print( "deleteDataAndMetaFromIndividualChunk : Chunk could not be deleted from node :", node) if (replicaNode != self.serverAddress and replicaNode in self.activeNodesChecker.getActiveChannels()): print("Deleting ") channel = active_ip_channel_dict[replicaNode] stub = fileService_pb2_grpc.FileserviceStub(channel) response = stub.FileDelete( fileService_pb2.FileInfo(username=username, filename=filename, seqNo=seqNo)) print(type(response.success)) if (response.success == True): print("Successfully deleted chunk from ReplicaNode : ", node) else: print("Chunk could not be deleted from ReplicaNode :", node)
def FileDelete(self, request, context): print("In FileDelete") if (self.fileExists(request.username, request.filename) == False): return fileService_pb2.ack( success=False, message="File {} does not exist.".format(request.filename)) fileMeta = db.parseMetaData(request.username, request.filename) print("FileMeta = ", fileMeta) primaryIP, replicaIP = -1, -1 channel1, channel2 = -1, -1 if (fileMeta[0] in self.clusterLeaders): primaryIP = self.clusterLeaders[fileMeta[0]] channel1 = self.clusterStatus.isChannelAlive(primaryIP) if (fileMeta[1] in self.clusterLeaders): replicaIP = self.clusterLeaders[fileMeta[1]] channel2 = self.clusterStatus.isChannelAlive(replicaIP) print("PrimarIP={}, replicaIP={}".format(primaryIP, replicaIP)) if (channel1 != -1): print("Making fileDelete call to primary") stub = fileService_pb2_grpc.FileserviceStub(channel1) response = stub.FileDelete( fileService_pb2.FileInfo(username=request.username, filename=request.filename)) print("Received response = ", response.success) if (channel2 != -1): print("Making fileDelete call to replica") stub = fileService_pb2_grpc.FileserviceStub(channel2) response = stub.FileDelete( fileService_pb2.FileInfo(username=request.username, filename=request.filename)) print("Received response = ", response.success) if (response.success == True): db.deleteEntry(request.username + "_" + request.filename) print("Successfully deleted.") return fileService_pb2.ack( success=True, message="File successfully deleted from cluster : " + fileMeta[0]) else: print("Could not delete from replica") return fileService_pb2.ack(success=False, message="Internal error")
def getDataFromIndividualNode(self, meta, username, filename): print("Inside getDataFromIndividualNode") print("Task Executed {}".format(threading.current_thread())) node, seqNo, replicaNode = str(meta[0]), meta[1], str(meta[2]) data = bytes("",'utf-8') result = {} if(node==str(self.serverAddress)): key = username + "_" + filename + "_" + str(seqNo) data = db.getFileData(key) else: if(node in self.active_ip_channel_dict): channel = self.active_ip_channel_dict[node] print("Fetching Data from Node {}".format(node)) elif(replicaNode in self.active_ip_channel_dict): channel = self.active_ip_channel_dict[replicaNode] print("Fetching Data from Node {}".format(replicaNode)) else: print("Both original and replica nodes are down!") return stub = fileService_pb2_grpc.FileserviceStub(channel) responses = stub.DownloadFile(fileService_pb2.FileInfo(username = username, filename = filename, seqNo = seqNo)) for response in responses: data+=response.data self.seqDataMap[seqNo] = data print("returning from the getDataFromIndividualNode")
def sendDataToDestination(self, chunk, node): self.activeNodeObj.channelRefresh() print(self.activeNodeObj.getActiveIpsDict()[node]) channel = self.activeNodeObj.getActiveIpsDict()[node] stub = fileService_pb2_grpc.FileserviceStub(channel) print('Calling next IP', self.activeNodeObj.isChannelAlive(channel)) return stub.UploadFile(self.dummy_generator(chunk))
def ReplicateFile(self, request, context): #print("request =", request.shortest_path[request.currentpos]) # # next_node = request.shortest_path[request.currentpos] # if request.currentpos == len(request.shortest_path) - 1: # cache.saveVClock(str(request), str(request)) # return fileService_pb2.ack(success=True, message="Data Replicated.") # else: # forward_server_addr = self.getneighbordata(next_node) #path = json.loads(request.shortest_path) forward_coordinates = request.shortest_path[request.currentpos] print("forward coord =", forward_coordinates) forward_server_addr = self.getneighbordata(forward_coordinates) print("forward IP =", forward_server_addr) forward_port = 50051 forward_channel = grpc.insecure_channel(forward_server_addr + ":" + str(forward_port)) forward_stub = fileService_pb2_grpc.FileserviceStub(forward_channel) request.currentpos += 1 rList = [1, 2, 3, 4, 5] arr = bytearray(rList) updated_request = fileService_pb2.FileData( initialReplicaServer=request.initialReplicaServer, bytearray=request.bytearray, vClock=request.vClock, shortest_path=request.shortest_path, currentpos=request.currentpos + 1) forward_resp = forward_stub.ReplicateFile(updated_request) print("forward_resp", forward_resp) return fileService_pb2.ack(success=True, message="Data Forwarded.")
def getLeaderInfo(self, request, context): channel = grpc.insecure_channel('{}'.format(self.superNodeAddress)) stub = fileService_pb2_grpc.FileserviceStub(channel) response = stub.getLeaderInfo( fileService_pb2.ClusterInfo(ip=self.hostname, port=self.serverPort, clusterName="team1")) print(response.message)
def broadcastMetadata(self, username, filename, chunk_id, destination): activeIpList = self.activeNodeObj.getActiveIpsDict() for ip, channel in activeIpList.items(): stub = fileService_pb2_grpc.FileserviceStub(channel) response = stub.metadataUpdate( fileService_pb2.metadataInfo(username=username, filename=filename, chunk_id=chunk_id, destination=destination))
def getDataFromNode(self, username, filename, chunk_id, destination): channel = grpc.insecure_channel(destination) channel2 = self.activeNodeObj.getActiveIpsDict()[destination] stub = fileService_pb2_grpc.FileserviceStub(channel2) response = stub.DownloadFile( fileService_pb2.FileInfo(username=username, filename=str(filename + chunk_id), sequence_no=str(chunk_id))) return response
def run_client(serverAddress): with grpc.insecure_channel(serverAddress) as channel: try: grpc.channel_ready_future(channel).result(timeout=1) except grpc.FutureTimeoutError: print("Connection timeout. Unable to connect to port ") #exit() else: print("Connected") stub = fileService_pb2_grpc.FileserviceStub(channel) handleUserInputs(stub)
def saveMetadataOnAllNodes(self, username, filename, metadata): print("saveMetadataOnAllNodes") active_ip_channel_dict = self.activeNodesChecker.getActiveChannels() uniqueFileName = username + "_" + filename for ip, channel in active_ip_channel_dict.items(): if (self.isChannelAlive(channel)): stub = fileService_pb2_grpc.FileserviceStub(channel) response = stub.MetaDataInfo( fileService_pb2.MetaData( filename=uniqueFileName, seqValues=str(metadata).encode('utf-8'))) print(response.message)
def sendLeaderInfoToSuperNode(self): try: channel = grpc.insecure_channel('{}'.format(self.superNodeAddress)) stub = fileService_pb2_grpc.FileserviceStub(channel) response = stub.getLeaderInfo( fileService_pb2.ClusterInfo(ip=self.hostname, port=self.serverPort, clusterName="team1")) print(response.message) except: print("Not able to connect to supernode") pass
def DownloadFile(self, request, context): # Check if file exists if (self.fileExists(request.username, request.filename) == False): return fileService_pb2.FileData(username=request.username, filename=request.filename, data=bytes("", 'utf-8')) fileMeta = db.parseMetaData(request.username, request.filename) primaryIP, replicaIP = -1, -1 channel1, channel2 = -1, -1 if (fileMeta[0] in self.clusterLeaders): primaryIP = self.clusterLeaders[fileMeta[0]] channel1 = self.clusterStatus.isChannelAlive(primaryIP) if (fileMeta[1] in self.clusterLeaders): replicaIP = self.clusterLeaders[fileMeta[1]] channel2 = self.clusterStatus.isChannelAlive(replicaIP) if (channel1): stub = fileService_pb2_grpc.FileserviceStub(channel1) responses = stub.DownloadFile( fileService_pb2.FileInfo(username=request.username, filename=request.filename)) for response in responses: yield response elif (channel2): stub = fileService_pb2_grpc.FileserviceStub(channel2) responses = stub.DownloadFile( fileService_pb2.FileInfo(username=request.username, filename=request.filename)) for response in responses: yield response else: return fileService_pb2.FileData(username=request.username, filename=request.filename, data=bytes("", 'utf-8'))
def client(): while True: choice = int( input( "What operation: 1. Upload 2. Download 3. Search 4. List File : " )) if choice == 1: channel = grpc.insecure_channel('192.168.43.74:9000') stub = fileService_pb2_grpc.FileserviceStub(channel) response = stub.getLeaderInfo( fileService_pb2.ClusterInfo(ip="192.168.43.2", port="9000", clusterName="akshay")) print(response.message)
def sendDataToDestination(self, currDataBytes, node, nodeReplica, username, filename, seqNo, channel): if (node == self.serverAddress): key = username + "_" + filename + "_" + str(seqNo) db.setData(key, currDataBytes) if (nodeReplica != ""): print("Sending replication to ", nodeReplica) active_ip_channel_dict = self.activeNodesChecker.getActiveChannels( ) replica_channel = active_ip_channel_dict[nodeReplica] stub = fileService_pb2_grpc.FileserviceStub(replica_channel) response = stub.UploadFile( self.sendDataInStream(currDataBytes, username, filename, seqNo, "")) return response else: print("Sending the UPLOAD_SHARD_SIZE to node :", node) stub = fileService_pb2_grpc.FileserviceStub(channel) response = stub.UploadFile( self.sendDataInStream(currDataBytes, username, filename, seqNo, nodeReplica)) print("Response from uploadFile: ", response.message) return response
def FileSearch(self, request, context): if (self.fileExists(request.username, request.filename) == False): return fileService_pb2.ack( success=False, message="File {} does not exist.".format(request.filename)) fileMeta = db.parseMetaData(request.username, request.filename) primaryIP, replicaIP = -1, -1 channel1, channel2 = -1, -1 if (fileMeta[0] in self.clusterLeaders): primaryIP = self.clusterLeaders[fileMeta[0]] channel1 = self.clusterStatus.isChannelAlive(primaryIP) if (fileMeta[1] in self.clusterLeaders): replicaIP = self.clusterLeaders[fileMeta[1]] channel2 = self.clusterStatus.isChannelAlive(replicaIP) if (channel1 != -1): stub = fileService_pb2_grpc.FileserviceStub(channel1) response = stub.FileSearch( fileService_pb2.FileInfo(username=request.username, filename=request.filename)) if (channel2 != -1): stub = fileService_pb2_grpc.FileserviceStub(channel2) response = stub.FileSearch( fileService_pb2.FileInfo(username=request.username, filename=request.filename)) if (response.success == True): return fileService_pb2.ack(success=True, message="File exists! ") else: return fileService_pb2.ack( success=False, message="File does not exist in any cluster.")
def client(): while True: choice = int(input("What operation: 1. Upload 2. Download")) if choice == 1: fileName = input("FileName to be uploaded: ") channel = grpc.insecure_channel('localhost:3000') stub = fileService_pb2_grpc.FileserviceStub(channel) response = stub.UploadFile(get_file_chunks(fileName)) print(response) elif choice == 2: name = input("Name of file to download") channel = grpc.insecure_channel('localhost:3000') stub = fluffy_pb2_grpc.FileserviceStub(channel) response = stub.DownloadFile(fluffy_pb2.FileInfo(fileName=name)) save_chunks_to_file(response, "downloads/" + name) print("File downloaded. ")
def UpdateFile(self, request_iterator, context): username, filename = "", "" fileData = bytes("", 'utf-8') for request in request_iterator: fileData += request.data username, filename = request.username, request.filename def getFileChunks(fileData): # Maximum chunk size that can be sent CHUNK_SIZE = 4000000 outfile = os.path.join('files', fileName) sTime = time.time() while True: chunk = fileData.read(CHUNK_SIZE) if not chunk: break yield fileService_pb2.FileData(username=username, filename=fileName, data=chunk, seqNo=1) print("Time for upload= ", time.time() - sTime) if (int(db.get("primaryStatus")) == 1): channel = grpc.insecure_channel('{}'.format(self.serverAddress)) stub = fileService_pb2_grpc.FileserviceStub(channel) response1 = stub.FileDelete( fileService_pb2.FileInfo(username=userName, filename=fileName)) if (response1.success): response2 = stub.UploadFile(getFileChunks(fileData)) if (response2.success): return fileService_pb2.ack( success=True, message="File suceessfully updated.") else: return fileService_pb2.ack(success=False, message="Internal error.") else: return fileService_pb2.ack(success=False, message="Internal error.")
def replicateContent(self, message, initialReplicaServer, address): # logic to pick up bytes from memory and transmit bytes_read_from_memory = str.encode("Srinivas") intial_Replicate_Server = self.localIP hostname = address serverAddress = "localhost" serverPort = 50051 channel = grpc.insecure_channel(serverAddress + ":" + str(serverPort)) replicate_stub = fileService_pb2_grpc.FileserviceStub(channel) if message.countOfReplica == 1: vClock = cache.getFileVclock("file1") vClock.ip2.address = hostname # vClock = { # "ip1": {"address": self.localIP, "timestamp": time.time()}, # "ip2": {"address": hostname, "timestamp": ""}, # "ip3": {"address": "", "timestamp": ""} # } else: vClock = cache.getFileVclock("file1") vClock.ip3.address = hostname file = "file1" message = json.dump({"vClock": vClock, "filename": file}, ) self.transmit_message(message, intial_Replicate_Server, vClock.ip2.address.decode("utf-8"), False, 0, "sync", "file1") self.transmit_message(message, intial_Replicate_Server, vClock.ip3.address.decode("utf-8"), False, 0, "sync", "file1") #print("vclock from redis", cache.getFileVclock("file1")) request = fileService_pb2.FileData( initialReplicaServer=intial_Replicate_Server, bytearray=bytes_read_from_memory, vClock=json.dumps(vClock)) resp = replicate_stub.ReplicateFile(request) print(resp)
def UploadFile(self, request_iterator, context): print("Inside Server method ---------- UploadFile") node, node_replica, clusterName, clusterReplica = self.clusterStatus.leastUtilizedNode( self.clusterLeaders) if (node == -1): return fileService_pb2.ack(success=False, message="No Active Clusters.") print("Node found is:{}, replica is:{}".format(node, node_replica)) channel1 = self.ip_channel_dict[node] stub1 = fileService_pb2_grpc.FileserviceStub(channel1) if (node_replica != "" and node_replica in self.ip_channel_dict): channel2 = self.ip_channel_dict[node_replica] stub2 = fileService_pb2_grpc.FileserviceStub(channel2) else: stub2 = None filename, username = "", "" data = bytes("", 'utf-8') for request in request_iterator: filename, username = request.filename, request.username data = request.data break if (self.fileExists(username, filename)): return fileService_pb2.ack( success=False, message= "File already exists for this user. Please rename or delete file first." ) def sendDataStreaming(username, filename, data): yield fileService_pb2.FileData(username=username, filename=filename, data=data) for request in request_iterator: data += request.data yield fileService_pb2.FileData(username=request.username, filename=request.filename, data=request.data) resp1 = stub1.UploadFile( sendDataStreaming(username, filename, request.data)) # Replicate to alternate cluster if (stub2 is not None): t1 = Thread(target=self.replicateData, args=( stub2, username, filename, data, )) t1.start() if (resp1.success): db.saveMetaData(username, filename, clusterName, clusterReplica) db.saveUserFile(username, filename) return resp1
import sys sys.path.append('./proto') import grpc import fileService_pb2 import fileService_pb2_grpc # open a gRPC channel channel = grpc.insecure_channel('localhost:50051') # create a stub (client) stub = fileService_pb2_grpc.FileserviceStub(channel) # create a valid request message request = fileService_pb2.FileData(initialReplicaServer="initialReplicaServer", message="message", vClock="vClock") # make the call resp = stub.ReplicateFile(request) # et voilĂ print(resp)
def sendDataToDestination(self, chunk, node, chunk_id): channel = self.activeNodeObj.getActiveIpsDict()[node] stub = fileService_pb2_grpc.FileserviceStub(channel) chunk_generator = self.getChunksinStream(chunk, chunk_id) response = stub.UploadFile(chunk_generator)
def replicateChunkData(self, replica_channel, dataToBeSaved, username, filename, sequenceNumberOfChunk): stub = fileService_pb2_grpc.FileserviceStub(replica_channel) response = stub.UploadFile( self.sendDataInStream(dataToBeSaved, username, filename, sequenceNumberOfChunk, ""))