def read_file(file_path, offset, numbytes):
master_addr = "localhost:{}".format(cfg.master_loc)
with grpc.insecure_channel(master_addr) as channel:
stub = gfs_pb2_grpc.MasterServerToClientStub(channel)
st = file_path + "|" + str(offset) + "|" + str(numbytes)
request = gfs_pb2.String(st=st)
master_response = stub.ReadFile(request).st
print("Response from master: {}".format(master_response))
if master_response.startswith("ERROR"):
return -1
file_content = ""
data = master_response.split("|")
for chunk_info in data:
chunk_handle, loc, start_offset, numbytes = chunk_info.split("*")
chunkserver_addr = "localhost:{}".format(loc)
with grpc.insecure_channel(chunkserver_addr) as channel:
stub = gfs_pb2_grpc.ChunkServerToClientStub(channel)
st = chunk_handle + "|" + start_offset + "|" + numbytes
request = gfs_pb2.String(st=st)
cs_response = stub.Read(request).st
print("Response from chunkserver {} : {}".format(loc, cs_response))
if cs_response.startswith("ERROR"):
return -1
file_content += cs_response
print(file_content)
def GetChunkSpace(self, request, context):
chunk_handle = request.st
print("{} GetChunkSpace {}".format(self.port, chunk_handle))
chunk_space, status = self.ckser.get_chunk_space(chunk_handle)
if status.v != 0:
return gfs_pb2.String(st=status.e)
else:
return gfs_pb2.String(st=chunk_space)
def AppendFile(self, request, context):
file_path = request.st
print("Command Append {}".format(file_path))
latest_chunk_handle, locs, status = self.master.append_file(file_path)
if status.v != 0:
return gfs_pb2.String(st=status.e)
st = latest_chunk_handle + "|" + "|".join(locs)
return gfs_pb2.String(st=st)
def CreateFile(self, request, context):
file_path = request.st
print("Command Create {}".format(file_path))
chunk_handle, locs, status = self.master.create_file(file_path)
if status.v != 0:
return gfs_pb2.String(st=status.e)
st = chunk_handle + "|" + "|".join(locs)
return gfs_pb2.String(st=st)
def undelete_file(file_path):
master_addr = "localhost:{}".format(cfg.master_loc)
with grpc.insecure_channel(master_addr) as channel:
stub = gfs_pb2_grpc.MasterServerToClientStub(channel)
request = gfs_pb2.String(st=file_path)
master_response = stub.UndeleteFile(request).st
print("Response from master: {}".format(master_response))
def list_files(file_path):
master = "localhost:{}".format(cfg.master_loc)
with grpc.insecure_channel(master) as channel:
stub = gfs_pb2_grpc.MasterServerToClientStub(channel)
request = gfs_pb2.String(st=file_path)
master_response = stub.ListFiles(request).st
fps = master_response.split("|")
print(fps)
def CreateChunk(self, request, context):
file_path, prev_chunk_handle = request.st.split("|")
print("Command CreateChunk {} {}".format(file_path, prev_chunk_handle))
chunk_handle, locs, status = self.master.create_chunk(
file_path, prev_chunk_handle)
# TODO: check status
st = chunk_handle + "|" + "|".join(locs)
return gfs_pb2.String(st=st)
def create_file(file_path):
master_addr = "localhost:{}".format(cfg.master_loc)
with grpc.insecure_channel(master_addr) as channel:
stub = gfs_pb2_grpc.MasterServerToClientStub(channel)
request = gfs_pb2.String(st=file_path)
master_response = stub.CreateFile(request).st
print("Response from master: {}".format(master_response))
if master_response.startswith("ERROR"):
return -1
data = master_response.split("|")
chunk_handle = data[0]
for loc in data[1:]:
chunkserver_addr = "localhost:{}".format(loc)
with grpc.insecure_channel(chunkserver_addr) as channel:
stub = gfs_pb2_grpc.ChunkServerToClientStub(channel)
request = gfs_pb2.String(st=chunk_handle)
cs_response = stub.Create(request).st
print("Response from chunkserver {} : {}".format(loc, cs_response))
def append_file(file_path, input_data):
master_addr = "localhost:{}".format(cfg.master_loc)
with grpc.insecure_channel(master_addr) as channel:
stub = gfs_pb2_grpc.MasterServerToClientStub(channel)
request = gfs_pb2.String(st=file_path)
master_response = stub.AppendFile(request).st
print("Response from master: {}".format(master_response))
if master_response.startswith("ERROR"):
return -1
input_size = len(input_data)
data = master_response.split("|")
chunk_handle = data[0]
for loc in data[1:]:
chunkserver_addr = "localhost:{}".format(loc)
with grpc.insecure_channel(chunkserver_addr) as channel:
stub = gfs_pb2_grpc.ChunkServerToClientStub(channel)
request = gfs_pb2.String(st=chunk_handle)
cs_response = stub.GetChunkSpace(request).st
print("Response from chunkserver {} : {}".format(loc, cs_response))
if cs_response.startswith("ERROR"):
return -1
rem_space = int(cs_response)
if rem_space >= input_size:
st = chunk_handle + "|" + input_data
request = gfs_pb2.String(st=st)
cs_response = stub.Append(request).st
print("Response from chunkserver {} : {}".format(loc, cs_response))
else:
inp1, inp2 = input_data[:rem_space], input_data[rem_space:]
st = chunk_handle + "|" + inp1
request = gfs_pb2.String(st=st)
cs_response = stub.Append(request).st
print("Response from chunkserver {} : {}".format(loc, cs_response))
if rem_space >= input_size:
return 0
# if need to add more chunks then continue
with grpc.insecure_channel(master_addr) as channel:
stub = gfs_pb2_grpc.MasterServerToClientStub(channel)
st = file_path + "|" + chunk_handle
request = gfs_pb2.String(st=st)
master_response = stub.CreateChunk(request).st
print("Response from master: {}".format(master_response))
data = master_response.split("|")
chunk_handle = data[0]
for loc in data[1:]:
chunkserver_addr = "localhost:{}".format(loc)
with grpc.insecure_channel(chunkserver_addr) as channel:
stub = gfs_pb2_grpc.ChunkServerToClientStub(channel)
request = gfs_pb2.String(st=chunk_handle)
cs_response = stub.Create(request).st
print("Response from chunkserver {} : {}".format(loc, cs_response))
append_file(file_path, inp2)
return 0
def Read(self, request, context):
chunk_handle, start_offset, numbytes = request.st.split("|")
print("{} Read {} {}".format(chunk_handle, start_offset, numbytes))
status = self.ckser.read(chunk_handle, start_offset, numbytes)
return gfs_pb2.String(st=status.e)
def Append(self, request, context):
chunk_handle, data = request.st.split("|")
print("{} Append {} {}".format(self.port, chunk_handle, data))
status = self.ckser.append(chunk_handle, data)
return gfs_pb2.String(st=status.e)
def Create(self, request, context):
chunk_handle = request.st
print("{} CreateChunk {}".format(self.port, chunk_handle))
status = self.ckser.create(chunk_handle)
return gfs_pb2.String(st=status.e)
def UndeleteFile(self, request, context):
file_path = request.st
print("Command Undelete {}".format(file_path))
status = self.master.undelete_file(file_path)
return gfs_pb2.String(st=status.e)
def ReadFile(self, request, context):
file_path, offset, numbytes = request.st.split("|")
print("Command ReadFile {} {} {}".format(file_path, offset, numbytes))
status = self.master.read_file(file_path, int(offset), int(numbytes))
return gfs_pb2.String(st=status.e)
def ListFiles(self, request, context):
file_path = request.st
print("Command List {}".format(file_path))
fpls = self.master.list_files(file_path)
st = "|".join(fpls)
return gfs_pb2.String(st=st)
