def DownloadFile(self, request_iterator, context):
activeIpList = self.activeNodeObj.getActiveIpsDict()
username = request_iterator.username
filename = request_iterator.filename
if (self.leader):
metadata = self.pickledbMetadataobj.getData(username, filename)
for item in metadata:
if item[1] == self.serverAddress:
fname = filename + str(item[0])
result = self.databaseHandlerObj.getData(username, fname)
yield fileService_pb2.FileData(username=result[0],
filename=filename,
data=result[2],
chunk_id=int(item[0]))
else:
data = self.getDataFromNode(username, filename, item[0],
item[1])
for d in data:
yield d
else:
result = self.databaseHandlerObj.getData(username, filename)
chunk_id = request_iterator.sequence_no
yield fileService_pb2.FileData(username=result[0],
filename=result[1][:-len(chunk_id)],
data=result[2],
chunk_id=int(chunk_id))
def sendDataStreaming(username, filename, data):
yield fileService_pb2.FileData(username=username,
filename=filename,
data=data)
for request in request_iterator:
data += request.data
yield fileService_pb2.FileData(username=request.username,
filename=request.filename,
data=request.data)
def ReplicateFile(self, request, context):
#print("request =", request.shortest_path[request.currentpos])
# # next_node = request.shortest_path[request.currentpos]
# if request.currentpos == len(request.shortest_path) - 1:
# cache.saveVClock(str(request), str(request))
# return fileService_pb2.ack(success=True, message="Data Replicated.")
# else:
# forward_server_addr = self.getneighbordata(next_node)
#path = json.loads(request.shortest_path)
forward_coordinates = request.shortest_path[request.currentpos]
print("forward coord =", forward_coordinates)
forward_server_addr = self.getneighbordata(forward_coordinates)
print("forward IP =", forward_server_addr)
forward_port = 50051
forward_channel = grpc.insecure_channel(forward_server_addr + ":" +
str(forward_port))
forward_stub = fileService_pb2_grpc.FileserviceStub(forward_channel)
request.currentpos += 1
rList = [1, 2, 3, 4, 5]
arr = bytearray(rList)
updated_request = fileService_pb2.FileData(
initialReplicaServer=request.initialReplicaServer,
bytearray=request.bytearray,
vClock=request.vClock,
shortest_path=request.shortest_path,
currentpos=request.currentpos + 1)
forward_resp = forward_stub.ReplicateFile(updated_request)
print("forward_resp", forward_resp)
return fileService_pb2.ack(success=True, message="Data Forwarded.")
def get_file_chunks(filename): with open(filename, 'rb') as f: while True: piece = f.read(CHUNK_SIZE) if len(piece) == 0: return yield fileService_pb2.FileData(username='******', filename= filename,data=piece)
def streamData(username, filename, data):
chunk_size = 4000000
start, end = 0, chunk_size
while (True):
chunk = data[start:end]
if (len(chunk) == 0): break
start = end
end += chunk_size
yield fileService_pb2.FileData(username=username,
filename=filename,
data=chunk)
def sendFileInChunks(username, filename, i):
# Maximum chunk size that can be sent
CHUNK_SIZE=4000000
outfile = os.path.join('files', fileName)
with open(outfile, 'rb') as infile:
while True:
chunk = infile.read(CHUNK_SIZE)
if not chunk: break
yield fileService_pb2.FileData(username=username+"_"+str(i), filename=fileName, data=chunk, seqNo=1)
def getChunksinStream(self, chunk, chunk_id):
end = sys.getsizeof(chunk.data)
start = 0
while True:
data = chunk.data[start:end]
if end > sys.getsizeof(chunk.data):
break
start = end
end = end + sys.getsizeof(chunk.data)
yield fileService_pb2.FileData(username=chunk.username,
filename=chunk.filename,
data=data,
chunk_id=chunk_id)
def DownloadFile(self, request, context):
# Check if file exists
if (self.fileExists(request.username, request.filename) == False):
return fileService_pb2.FileData(username=request.username,
filename=request.filename,
data=bytes("", 'utf-8'))
fileMeta = db.parseMetaData(request.username, request.filename)
primaryIP, replicaIP = -1, -1
channel1, channel2 = -1, -1
if (fileMeta[0] in self.clusterLeaders):
primaryIP = self.clusterLeaders[fileMeta[0]]
channel1 = self.clusterStatus.isChannelAlive(primaryIP)
if (fileMeta[1] in self.clusterLeaders):
replicaIP = self.clusterLeaders[fileMeta[1]]
channel2 = self.clusterStatus.isChannelAlive(replicaIP)
if (channel1):
stub = fileService_pb2_grpc.FileserviceStub(channel1)
responses = stub.DownloadFile(
fileService_pb2.FileInfo(username=request.username,
filename=request.filename))
for response in responses:
yield response
elif (channel2):
stub = fileService_pb2_grpc.FileserviceStub(channel2)
responses = stub.DownloadFile(
fileService_pb2.FileInfo(username=request.username,
filename=request.filename))
for response in responses:
yield response
else:
return fileService_pb2.FileData(username=request.username,
filename=request.filename,
data=bytes("", 'utf-8'))
def sendDataInStream(self, dataBytes, username, filename, seqNo,
replicaNode):
chunk_size = 4000000
start, end = 0, chunk_size
while (True):
chunk = dataBytes[start:end]
if (len(chunk) == 0): break
start = end
end += chunk_size
yield fileService_pb2.FileData(username=username,
filename=filename,
data=chunk,
seqNo=seqNo,
replicaNode=replicaNode)
def getFileChunks(fileData):
# Maximum chunk size that can be sent
CHUNK_SIZE = 4000000
outfile = os.path.join('files', fileName)
sTime = time.time()
while True:
chunk = fileData.read(CHUNK_SIZE)
if not chunk: break
yield fileService_pb2.FileData(username=username,
filename=fileName,
data=chunk,
seqNo=1)
print("Time for upload= ", time.time() - sTime)
def getFileChunks():
# Maximum chunk size that can be sent
CHUNK_SIZE=4000000
username = input("Enter Username: "******"Enter filename: ")
outfile = os.path.join('files', fileName)
sTime=time.time()
with open(outfile, 'rb') as infile:
while True:
chunk = infile.read(CHUNK_SIZE)
if not chunk: break
# Do what you want with each chunk (in dev, write line to file)
yield fileService_pb2.FileData(username=username, filename=fileName, data=chunk, seqNo=1)
print("Time for upload= ", time.time()-sTime)
def replicateContent(self, message, initialReplicaServer, address):
# logic to pick up bytes from memory and transmit
bytes_read_from_memory = str.encode("Srinivas")
intial_Replicate_Server = self.localIP
hostname = address
serverAddress = "localhost"
serverPort = 50051
channel = grpc.insecure_channel(serverAddress + ":" + str(serverPort))
replicate_stub = fileService_pb2_grpc.FileserviceStub(channel)
if message.countOfReplica == 1:
vClock = cache.getFileVclock("file1")
vClock.ip2.address = hostname
# vClock = {
# "ip1": {"address": self.localIP, "timestamp": time.time()},
# "ip2": {"address": hostname, "timestamp": ""},
# "ip3": {"address": "", "timestamp": ""}
# }
else:
vClock = cache.getFileVclock("file1")
vClock.ip3.address = hostname
file = "file1"
message = json.dump({"vClock": vClock, "filename": file}, )
self.transmit_message(message, intial_Replicate_Server,
vClock.ip2.address.decode("utf-8"), False, 0,
"sync", "file1")
self.transmit_message(message, intial_Replicate_Server,
vClock.ip3.address.decode("utf-8"), False, 0,
"sync", "file1")
#print("vclock from redis", cache.getFileVclock("file1"))
request = fileService_pb2.FileData(
initialReplicaServer=intial_Replicate_Server,
bytearray=bytes_read_from_memory,
vClock=json.dumps(vClock))
resp = replicate_stub.ReplicateFile(request)
print(resp)
def dummy_generator(self, chunk):
yield fileService_pb2.FileData(username=chunk.username,
filename=chunk.filename,
data=chunk.data)
def DownloadFile(self, request, context):
print("Inside Download")
# If the node is the leader of the cluster.
if (int(db.get("primaryStatus")) == 1):
print("Inside primary download")
# Check if file exists
if (self.fileExists(request.username, request.filename) == 0):
print("File does not exist")
yield fileService_pb2.FileData(username=request.username,
filename=request.filename,
data=bytes("", 'utf-8'),
seqNo=0)
return
# If the file is present in cache then just fetch it and return. No need to go to individual node.
if (self.lru.has_key(request.username + "_" + request.filename)):
print("Fetching data from Cache")
CHUNK_SIZE = 4000000
fileName = request.username + "_" + request.filename
filePath = self.lru[fileName]
outfile = os.path.join(filePath, fileName)
with open(outfile, 'rb') as infile:
while True:
chunk = infile.read(CHUNK_SIZE)
if not chunk: break
yield fileService_pb2.FileData(
username=request.username,
filename=request.filename,
data=chunk,
seqNo=1)
# If the file is not present in the cache, then fetch it from the individual node.
else:
print("Fetching the metadata")
# Step 1: get metadata i.e. the location of chunks.
metaData = db.parseMetaData(request.username, request.filename)
print(metaData)
#Step 2: make gRPC calls and get the fileData from all the nodes.
downloadHelper = DownloadHelper(self.hostname, self.serverPort,
self.activeNodesChecker)
data = downloadHelper.getDataFromNodes(request.username,
request.filename,
metaData)
print("Sending the data to client")
#Step 3: send the file to supernode using gRPC streaming.
chunk_size = 4000000
start, end = 0, chunk_size
while (True):
chunk = data[start:end]
if (len(chunk) == 0): break
start = end
end += chunk_size
yield fileService_pb2.FileData(username=request.username,
filename=request.filename,
data=chunk,
seqNo=request.seqNo)
# Step 4: update the cache based on LRU(least recently used) algorithm.
self.saveInCache(request.username, request.filename, data)
# If the node is not the leader, then just fetch the fileChunk from the local db and stream it back to leader.
else:
key = request.username + "_" + request.filename + "_" + str(
request.seqNo)
print(key)
data = db.getFileData(key)
chunk_size = 4000000
start, end = 0, chunk_size
while (True):
chunk = data[start:end]
if (len(chunk) == 0): break
start = end
end += chunk_size
yield fileService_pb2.FileData(username=request.username,
filename=request.filename,
data=chunk,
seqNo=request.seqNo)
import sys
sys.path.append('./proto')
import grpc
import fileService_pb2
import fileService_pb2_grpc
# open a gRPC channel
channel = grpc.insecure_channel('localhost:50051')
# create a stub (client)
stub = fileService_pb2_grpc.FileserviceStub(channel)
# create a valid request message
request = fileService_pb2.FileData(initialReplicaServer="initialReplicaServer", message="message",
vClock="vClock")
# make the call
resp = stub.ReplicateFile(request)
# et voilà
print(resp)
def getFileData():
fileName = input("Enter filename:")
outfile = os.path.join('files', fileName)
file_data = open(outfile, 'rb').read()
fileData = fileService_pb2.FileData(fileName=fileName, data=file_data)
return fileData