def client():
while True:
choice= int(input("What operation: 1. Upload 2. Download 3. Search 4. List File : "))
if choice==1:
channel = grpc.insecure_channel('127.0.0.1:3000')
fileName=input("FileName to be uploaded: ")
chunk_generator= get_file_chunks(fileName)
stub = fileService_pb2_grpc.FileserviceStub(channel)
response = stub.UploadFile(chunk_generator)
print(response)
elif choice==2:
channel = grpc.insecure_channel('127.0.0.1:3000')
name= input("Name of file to download")
stub = fileService_pb2_grpc.FileserviceStub(channel)
response = stub.DownloadFile(fileService_pb2.FileInfo(username="******", filename=name))
save_chunks_to_file(response, "downloads/own.pdf")
print("File downloaded. ")
elif choice==3:
channel = grpc.insecure_channel('127.0.0.1:3000')
name= input("Name of file to Search")
stub = fileService_pb2_grpc.FileserviceStub(channel)
response = stub.FileSearch(fileService_pb2.FileInfo(username="******", filename=name))
print(response)
elif choice==4:
channel = grpc.insecure_channel('127.0.0.1:3000')
name= input("Enter username: "******"akshay", filename=name))
print("File List. ", fileList)
def client():
while True:
choice= int(input("What operation: 1. Upload 2. Download 3. File Search 4. File List 5. File Delete : "))
if choice==1:
channel = grpc.insecure_channel('169.254.46.132:3000')
fileName=input("FileName to be uploaded: ")
# start = timer()
start_time = time.time()
chunk_generator= get_file_chunks(fileName)
# time.sleep(1)
# end = timer()
# print(end-start)
# print("Chunks generated in %s seconds" % (time.time() - start_time))
print("Chunks generated in %s seconds" % (str((time.time() - start_time))))
start_time = time.time()
stub = fileService_pb2_grpc.FileserviceStub(channel)
response = stub.UploadFile(chunk_generator)
print("File Uploaded in - %s seconds" % (str(round(time.time() - start_time,10))))
print(response)
elif choice==2:
channel = grpc.insecure_channel('169.254.46.132:3000')
name= input("Name of file to download")
start_time = time.time()
stub = fileService_pb2_grpc.FileserviceStub(channel)
res = stub.FileSearch(fileService_pb2.FileInfo(username="******", filename=name))
if res.success:
response = stub.DownloadFile(fileService_pb2.FileInfo(username="******", filename=name))
print("Files downloaded in %s seconds" % (str(round(time.time() - start_time,10))))
start_time = time.time()
save_chunks_to_file(response, "downloads/"+name)
print("Download chunks aggregated in %s seconds" % (str(round(time.time() - start_time,10))))
print("File downloaded. ")
else:
print("File not found")
elif choice==3:
channel = grpc.insecure_channel('169.254.46.132:3000')
name= input("Name of file to Search")
stub = fileService_pb2_grpc.FileserviceStub(channel)
response = stub.FileSearch(fileService_pb2.FileInfo(username="******", filename=name))
print(response)
elif choice==4:
channel = grpc.insecure_channel('169.254.46.132:3000')
name= input("Enter username: "******"akshay", filename=name))
print("File List. ", fileList)
elif choice==5:
channel = grpc.insecure_channel('169.254.46.132:3000')
name= input("Name of file to Delete: ")
stub = fileService_pb2_grpc.FileserviceStub(channel)
res = stub.FileSearch(fileService_pb2.FileInfo(username="******", filename=name))
if res.success:
response= stub.FileDelete(fileService_pb2.FileInfo(username="******", filename=name))
print(response)
else:
print("file not found")
def deleteDataAndMetaFromIndividualChunk(self, meta, username, filename):
node, seqNo, replicaNode = str(meta[0]), meta[1], str(meta[2])
metaDataKey = username + "_" + filename
dataChunkKey = username + "_" + filename + "_" + str(seqNo)
if (db.keyExists(metaDataKey) == 1):
print(
"deleteDataAndMetaFromIndividualChunk: Deleting the metadataEntry from local db :",
node)
db.deleteEntry(metaDataKey)
if (db.keyExists(dataChunkKey)):
print(
"deleteDataAndMetaFromIndividualChunk: Deleting the data chunk from local db:",
node)
db.deleteEntry(dataChunkKey)
active_ip_channel_dict = self.activeNodesChecker.getActiveChannels()
if (node != self.serverAddress and node in active_ip_channel_dict):
channel = active_ip_channel_dict[node]
stub = fileService_pb2_grpc.FileserviceStub(channel)
response = stub.FileDelete(
fileService_pb2.FileInfo(username=username,
filename=filename,
seqNo=seqNo))
if (response.success == True):
print(
"deleteDataAndMetaFromIndividualChunk : Successfully deleted chunk from node : ",
node)
else:
print(
"deleteDataAndMetaFromIndividualChunk : Chunk could not be deleted from node :",
node)
if (replicaNode != self.serverAddress and replicaNode
in self.activeNodesChecker.getActiveChannels()):
print("Deleting ")
channel = active_ip_channel_dict[replicaNode]
stub = fileService_pb2_grpc.FileserviceStub(channel)
response = stub.FileDelete(
fileService_pb2.FileInfo(username=username,
filename=filename,
seqNo=seqNo))
print(type(response.success))
if (response.success == True):
print("Successfully deleted chunk from ReplicaNode : ", node)
else:
print("Chunk could not be deleted from ReplicaNode :", node)
def FileDelete(self, request, context):
print("In FileDelete")
if (self.fileExists(request.username, request.filename) == False):
return fileService_pb2.ack(
success=False,
message="File {} does not exist.".format(request.filename))
fileMeta = db.parseMetaData(request.username, request.filename)
print("FileMeta = ", fileMeta)
primaryIP, replicaIP = -1, -1
channel1, channel2 = -1, -1
if (fileMeta[0] in self.clusterLeaders):
primaryIP = self.clusterLeaders[fileMeta[0]]
channel1 = self.clusterStatus.isChannelAlive(primaryIP)
if (fileMeta[1] in self.clusterLeaders):
replicaIP = self.clusterLeaders[fileMeta[1]]
channel2 = self.clusterStatus.isChannelAlive(replicaIP)
print("PrimarIP={}, replicaIP={}".format(primaryIP, replicaIP))
if (channel1 != -1):
print("Making fileDelete call to primary")
stub = fileService_pb2_grpc.FileserviceStub(channel1)
response = stub.FileDelete(
fileService_pb2.FileInfo(username=request.username,
filename=request.filename))
print("Received response = ", response.success)
if (channel2 != -1):
print("Making fileDelete call to replica")
stub = fileService_pb2_grpc.FileserviceStub(channel2)
response = stub.FileDelete(
fileService_pb2.FileInfo(username=request.username,
filename=request.filename))
print("Received response = ", response.success)
if (response.success == True):
db.deleteEntry(request.username + "_" + request.filename)
print("Successfully deleted.")
return fileService_pb2.ack(
success=True,
message="File successfully deleted from cluster : " +
fileMeta[0])
else:
print("Could not delete from replica")
return fileService_pb2.ack(success=False, message="Internal error")
def getDataFromIndividualNode(self, meta, username, filename):
print("Inside getDataFromIndividualNode")
print("Task Executed {}".format(threading.current_thread()))
node, seqNo, replicaNode = str(meta[0]), meta[1], str(meta[2])
data = bytes("",'utf-8')
result = {}
if(node==str(self.serverAddress)):
key = username + "_" + filename + "_" + str(seqNo)
data = db.getFileData(key)
else:
if(node in self.active_ip_channel_dict):
channel = self.active_ip_channel_dict[node]
print("Fetching Data from Node {}".format(node))
elif(replicaNode in self.active_ip_channel_dict):
channel = self.active_ip_channel_dict[replicaNode]
print("Fetching Data from Node {}".format(replicaNode))
else:
print("Both original and replica nodes are down!")
return
stub = fileService_pb2_grpc.FileserviceStub(channel)
responses = stub.DownloadFile(fileService_pb2.FileInfo(username = username, filename = filename, seqNo = seqNo))
for response in responses:
data+=response.data
self.seqDataMap[seqNo] = data
print("returning from the getDataFromIndividualNode")
def sendDataToDestination(self, chunk, node):
self.activeNodeObj.channelRefresh()
print(self.activeNodeObj.getActiveIpsDict()[node])
channel = self.activeNodeObj.getActiveIpsDict()[node]
stub = fileService_pb2_grpc.FileserviceStub(channel)
print('Calling next IP', self.activeNodeObj.isChannelAlive(channel))
return stub.UploadFile(self.dummy_generator(chunk))
def ReplicateFile(self, request, context):
#print("request =", request.shortest_path[request.currentpos])
# # next_node = request.shortest_path[request.currentpos]
# if request.currentpos == len(request.shortest_path) - 1:
# cache.saveVClock(str(request), str(request))
# return fileService_pb2.ack(success=True, message="Data Replicated.")
# else:
# forward_server_addr = self.getneighbordata(next_node)
#path = json.loads(request.shortest_path)
forward_coordinates = request.shortest_path[request.currentpos]
print("forward coord =", forward_coordinates)
forward_server_addr = self.getneighbordata(forward_coordinates)
print("forward IP =", forward_server_addr)
forward_port = 50051
forward_channel = grpc.insecure_channel(forward_server_addr + ":" +
str(forward_port))
forward_stub = fileService_pb2_grpc.FileserviceStub(forward_channel)
request.currentpos += 1
rList = [1, 2, 3, 4, 5]
arr = bytearray(rList)
updated_request = fileService_pb2.FileData(
initialReplicaServer=request.initialReplicaServer,
bytearray=request.bytearray,
vClock=request.vClock,
shortest_path=request.shortest_path,
currentpos=request.currentpos + 1)
forward_resp = forward_stub.ReplicateFile(updated_request)
print("forward_resp", forward_resp)
return fileService_pb2.ack(success=True, message="Data Forwarded.")
def getLeaderInfo(self, request, context):
channel = grpc.insecure_channel('{}'.format(self.superNodeAddress))
stub = fileService_pb2_grpc.FileserviceStub(channel)
response = stub.getLeaderInfo(
fileService_pb2.ClusterInfo(ip=self.hostname,
port=self.serverPort,
clusterName="team1"))
print(response.message)
def broadcastMetadata(self, username, filename, chunk_id, destination):
activeIpList = self.activeNodeObj.getActiveIpsDict()
for ip, channel in activeIpList.items():
stub = fileService_pb2_grpc.FileserviceStub(channel)
response = stub.metadataUpdate(
fileService_pb2.metadataInfo(username=username,
filename=filename,
chunk_id=chunk_id,
destination=destination))
def getDataFromNode(self, username, filename, chunk_id, destination):
channel = grpc.insecure_channel(destination)
channel2 = self.activeNodeObj.getActiveIpsDict()[destination]
stub = fileService_pb2_grpc.FileserviceStub(channel2)
response = stub.DownloadFile(
fileService_pb2.FileInfo(username=username,
filename=str(filename + chunk_id),
sequence_no=str(chunk_id)))
return response
def run_client(serverAddress):
with grpc.insecure_channel(serverAddress) as channel:
try:
grpc.channel_ready_future(channel).result(timeout=1)
except grpc.FutureTimeoutError:
print("Connection timeout. Unable to connect to port ")
#exit()
else:
print("Connected")
stub = fileService_pb2_grpc.FileserviceStub(channel)
handleUserInputs(stub)
def saveMetadataOnAllNodes(self, username, filename, metadata):
print("saveMetadataOnAllNodes")
active_ip_channel_dict = self.activeNodesChecker.getActiveChannels()
uniqueFileName = username + "_" + filename
for ip, channel in active_ip_channel_dict.items():
if (self.isChannelAlive(channel)):
stub = fileService_pb2_grpc.FileserviceStub(channel)
response = stub.MetaDataInfo(
fileService_pb2.MetaData(
filename=uniqueFileName,
seqValues=str(metadata).encode('utf-8')))
print(response.message)
def sendLeaderInfoToSuperNode(self):
try:
channel = grpc.insecure_channel('{}'.format(self.superNodeAddress))
stub = fileService_pb2_grpc.FileserviceStub(channel)
response = stub.getLeaderInfo(
fileService_pb2.ClusterInfo(ip=self.hostname,
port=self.serverPort,
clusterName="team1"))
print(response.message)
except:
print("Not able to connect to supernode")
pass
def DownloadFile(self, request, context):
# Check if file exists
if (self.fileExists(request.username, request.filename) == False):
return fileService_pb2.FileData(username=request.username,
filename=request.filename,
data=bytes("", 'utf-8'))
fileMeta = db.parseMetaData(request.username, request.filename)
primaryIP, replicaIP = -1, -1
channel1, channel2 = -1, -1
if (fileMeta[0] in self.clusterLeaders):
primaryIP = self.clusterLeaders[fileMeta[0]]
channel1 = self.clusterStatus.isChannelAlive(primaryIP)
if (fileMeta[1] in self.clusterLeaders):
replicaIP = self.clusterLeaders[fileMeta[1]]
channel2 = self.clusterStatus.isChannelAlive(replicaIP)
if (channel1):
stub = fileService_pb2_grpc.FileserviceStub(channel1)
responses = stub.DownloadFile(
fileService_pb2.FileInfo(username=request.username,
filename=request.filename))
for response in responses:
yield response
elif (channel2):
stub = fileService_pb2_grpc.FileserviceStub(channel2)
responses = stub.DownloadFile(
fileService_pb2.FileInfo(username=request.username,
filename=request.filename))
for response in responses:
yield response
else:
return fileService_pb2.FileData(username=request.username,
filename=request.filename,
data=bytes("", 'utf-8'))
def client():
while True:
choice = int(
input(
"What operation: 1. Upload 2. Download 3. Search 4. List File : "
))
if choice == 1:
channel = grpc.insecure_channel('192.168.43.74:9000')
stub = fileService_pb2_grpc.FileserviceStub(channel)
response = stub.getLeaderInfo(
fileService_pb2.ClusterInfo(ip="192.168.43.2",
port="9000",
clusterName="akshay"))
print(response.message)
def sendDataToDestination(self, currDataBytes, node, nodeReplica, username,
filename, seqNo, channel):
if (node == self.serverAddress):
key = username + "_" + filename + "_" + str(seqNo)
db.setData(key, currDataBytes)
if (nodeReplica != ""):
print("Sending replication to ", nodeReplica)
active_ip_channel_dict = self.activeNodesChecker.getActiveChannels(
)
replica_channel = active_ip_channel_dict[nodeReplica]
stub = fileService_pb2_grpc.FileserviceStub(replica_channel)
response = stub.UploadFile(
self.sendDataInStream(currDataBytes, username, filename,
seqNo, ""))
return response
else:
print("Sending the UPLOAD_SHARD_SIZE to node :", node)
stub = fileService_pb2_grpc.FileserviceStub(channel)
response = stub.UploadFile(
self.sendDataInStream(currDataBytes, username, filename, seqNo,
nodeReplica))
print("Response from uploadFile: ", response.message)
return response
def FileSearch(self, request, context):
if (self.fileExists(request.username, request.filename) == False):
return fileService_pb2.ack(
success=False,
message="File {} does not exist.".format(request.filename))
fileMeta = db.parseMetaData(request.username, request.filename)
primaryIP, replicaIP = -1, -1
channel1, channel2 = -1, -1
if (fileMeta[0] in self.clusterLeaders):
primaryIP = self.clusterLeaders[fileMeta[0]]
channel1 = self.clusterStatus.isChannelAlive(primaryIP)
if (fileMeta[1] in self.clusterLeaders):
replicaIP = self.clusterLeaders[fileMeta[1]]
channel2 = self.clusterStatus.isChannelAlive(replicaIP)
if (channel1 != -1):
stub = fileService_pb2_grpc.FileserviceStub(channel1)
response = stub.FileSearch(
fileService_pb2.FileInfo(username=request.username,
filename=request.filename))
if (channel2 != -1):
stub = fileService_pb2_grpc.FileserviceStub(channel2)
response = stub.FileSearch(
fileService_pb2.FileInfo(username=request.username,
filename=request.filename))
if (response.success == True):
return fileService_pb2.ack(success=True, message="File exists! ")
else:
return fileService_pb2.ack(
success=False, message="File does not exist in any cluster.")
def client():
while True:
choice = int(input("What operation: 1. Upload 2. Download"))
if choice == 1:
fileName = input("FileName to be uploaded: ")
channel = grpc.insecure_channel('localhost:3000')
stub = fileService_pb2_grpc.FileserviceStub(channel)
response = stub.UploadFile(get_file_chunks(fileName))
print(response)
elif choice == 2:
name = input("Name of file to download")
channel = grpc.insecure_channel('localhost:3000')
stub = fluffy_pb2_grpc.FileserviceStub(channel)
response = stub.DownloadFile(fluffy_pb2.FileInfo(fileName=name))
save_chunks_to_file(response, "downloads/" + name)
print("File downloaded. ")
def UpdateFile(self, request_iterator, context):
username, filename = "", ""
fileData = bytes("", 'utf-8')
for request in request_iterator:
fileData += request.data
username, filename = request.username, request.filename
def getFileChunks(fileData):
# Maximum chunk size that can be sent
CHUNK_SIZE = 4000000
outfile = os.path.join('files', fileName)
sTime = time.time()
while True:
chunk = fileData.read(CHUNK_SIZE)
if not chunk: break
yield fileService_pb2.FileData(username=username,
filename=fileName,
data=chunk,
seqNo=1)
print("Time for upload= ", time.time() - sTime)
if (int(db.get("primaryStatus")) == 1):
channel = grpc.insecure_channel('{}'.format(self.serverAddress))
stub = fileService_pb2_grpc.FileserviceStub(channel)
response1 = stub.FileDelete(
fileService_pb2.FileInfo(username=userName, filename=fileName))
if (response1.success):
response2 = stub.UploadFile(getFileChunks(fileData))
if (response2.success):
return fileService_pb2.ack(
success=True, message="File suceessfully updated.")
else:
return fileService_pb2.ack(success=False,
message="Internal error.")
else:
return fileService_pb2.ack(success=False,
message="Internal error.")
def replicateContent(self, message, initialReplicaServer, address):
# logic to pick up bytes from memory and transmit
bytes_read_from_memory = str.encode("Srinivas")
intial_Replicate_Server = self.localIP
hostname = address
serverAddress = "localhost"
serverPort = 50051
channel = grpc.insecure_channel(serverAddress + ":" + str(serverPort))
replicate_stub = fileService_pb2_grpc.FileserviceStub(channel)
if message.countOfReplica == 1:
vClock = cache.getFileVclock("file1")
vClock.ip2.address = hostname
# vClock = {
# "ip1": {"address": self.localIP, "timestamp": time.time()},
# "ip2": {"address": hostname, "timestamp": ""},
# "ip3": {"address": "", "timestamp": ""}
# }
else:
vClock = cache.getFileVclock("file1")
vClock.ip3.address = hostname
file = "file1"
message = json.dump({"vClock": vClock, "filename": file}, )
self.transmit_message(message, intial_Replicate_Server,
vClock.ip2.address.decode("utf-8"), False, 0,
"sync", "file1")
self.transmit_message(message, intial_Replicate_Server,
vClock.ip3.address.decode("utf-8"), False, 0,
"sync", "file1")
#print("vclock from redis", cache.getFileVclock("file1"))
request = fileService_pb2.FileData(
initialReplicaServer=intial_Replicate_Server,
bytearray=bytes_read_from_memory,
vClock=json.dumps(vClock))
resp = replicate_stub.ReplicateFile(request)
print(resp)
def UploadFile(self, request_iterator, context):
print("Inside Server method ---------- UploadFile")
node, node_replica, clusterName, clusterReplica = self.clusterStatus.leastUtilizedNode(
self.clusterLeaders)
if (node == -1):
return fileService_pb2.ack(success=False,
message="No Active Clusters.")
print("Node found is:{}, replica is:{}".format(node, node_replica))
channel1 = self.ip_channel_dict[node]
stub1 = fileService_pb2_grpc.FileserviceStub(channel1)
if (node_replica != "" and node_replica in self.ip_channel_dict):
channel2 = self.ip_channel_dict[node_replica]
stub2 = fileService_pb2_grpc.FileserviceStub(channel2)
else:
stub2 = None
filename, username = "", ""
data = bytes("", 'utf-8')
for request in request_iterator:
filename, username = request.filename, request.username
data = request.data
break
if (self.fileExists(username, filename)):
return fileService_pb2.ack(
success=False,
message=
"File already exists for this user. Please rename or delete file first."
)
def sendDataStreaming(username, filename, data):
yield fileService_pb2.FileData(username=username,
filename=filename,
data=data)
for request in request_iterator:
data += request.data
yield fileService_pb2.FileData(username=request.username,
filename=request.filename,
data=request.data)
resp1 = stub1.UploadFile(
sendDataStreaming(username, filename, request.data))
# Replicate to alternate cluster
if (stub2 is not None):
t1 = Thread(target=self.replicateData,
args=(
stub2,
username,
filename,
data,
))
t1.start()
if (resp1.success):
db.saveMetaData(username, filename, clusterName, clusterReplica)
db.saveUserFile(username, filename)
return resp1
import sys
sys.path.append('./proto')
import grpc
import fileService_pb2
import fileService_pb2_grpc
# open a gRPC channel
channel = grpc.insecure_channel('localhost:50051')
# create a stub (client)
stub = fileService_pb2_grpc.FileserviceStub(channel)
# create a valid request message
request = fileService_pb2.FileData(initialReplicaServer="initialReplicaServer", message="message",
vClock="vClock")
# make the call
resp = stub.ReplicateFile(request)
# et voilà
print(resp)
def sendDataToDestination(self, chunk, node, chunk_id):
channel = self.activeNodeObj.getActiveIpsDict()[node]
stub = fileService_pb2_grpc.FileserviceStub(channel)
chunk_generator = self.getChunksinStream(chunk, chunk_id)
response = stub.UploadFile(chunk_generator)
def replicateChunkData(self, replica_channel, dataToBeSaved, username,
filename, sequenceNumberOfChunk):
stub = fileService_pb2_grpc.FileserviceStub(replica_channel)
response = stub.UploadFile(
self.sendDataInStream(dataToBeSaved, username, filename,
sequenceNumberOfChunk, ""))
