def append(fname, data): "append a block of data to a file" def log(str): _log("[client:append] " + str) log("append(%s,len=%i )" % (fname, len(data))) log("talking to master") append_req = msg.AppendReq(fname, len(data)) master_comm = net.PakComm( (settings.MASTER_ADDR, settings.MASTER_CLIENT_PORT), "client:append") for res in msg.handle_req_response(master_comm, append_req, log): if res: break yield None if not res: log("no response from master on append request. failing") return chunk_info = res # TODO another deviation: in real GFS, 'closest' chunkserver would be # picked using some heuristic random.shuffle(chunk_info.servers) chunkaddr, port = chunk_info.servers[0] log("mutate id %i from master. connecting to chunkserver (%s,%i) (picked from %s)" % (chunk_info.mutate_id, chunkaddr, port, str(chunk_info.servers))) chunk_comm = net.PakComm((chunkaddr, port), "client:append") write_req = msg.SendData(chunk_info, data) for res in msg.handle_req_response(chunk_comm, write_req, log): if res: break yield None if not res: log("no response from chunkservers on Senddata. failing") return log("append %i succeeded got %s from chunkserver" % (res.mutate_id, str(res))) commit_req = msg.CommitAppendReq(chunk_info) for res in msg.handle_req_response(master_comm, commit_req, log): if res: break yield None if not res: log("no respone from master on commit. failing") return # TODO any kind of retrying log("%i write complete " % res.mutate_id) yield res
def append(fname, data): "append a block of data to a file" def log(str): _log("[client:append] " + str) log("append(%s,len=%i )" % (fname, len(data))) log("talking to master") append_req = msg.AppendReq(fname, len(data)) master_comm = net.PakComm((settings.MASTER_ADDR, settings.MASTER_CLIENT_PORT), "client:append") for res in msg.handle_req_response(master_comm, append_req, log): if res: break yield None if not res: log("no response from master on append request. failing") return chunk_info = res # TODO another deviation: in real GFS, 'closest' chunkserver would be # picked using some heuristic random.shuffle(chunk_info.servers) chunkaddr, port = chunk_info.servers[0] log( "mutate id %i from master. connecting to chunkserver (%s,%i) (picked from %s)" % (chunk_info.mutate_id, chunkaddr, port, str(chunk_info.servers)) ) chunk_comm = net.PakComm((chunkaddr, port), "client:append") write_req = msg.SendData(chunk_info, data) for res in msg.handle_req_response(chunk_comm, write_req, log): if res: break yield None if not res: log("no response from chunkservers on Senddata. failing") return log("append %i succeeded got %s from chunkserver" % (res.mutate_id, str(res))) commit_req = msg.CommitAppendReq(chunk_info) for res in msg.handle_req_response(master_comm, commit_req, log): if res: break yield None if not res: log("no respone from master on commit. failing") return # TODO any kind of retrying log("%i write complete " % res.mutate_id) yield res
def read(fname, offset, len): """involves these steps: 1 request (fname/offset) from the master 1a master puts read lock on fname 2 receive a chunk handle and set of chunkservers with chunk 3 pick a chunkserver (nearest), request the data 4 close handle on the master 4a master releases lock """ def log(str): _log("[client:read] " + str) log( "read(%s,%i,%i), connecting to (%s,%i)" % (fname, offset, len, settings.MASTER_ADDR, settings.MASTER_CLIENT_PORT) ) log("connecting to master") master_comm = net.PakComm((settings.MASTER_ADDR, settings.MASTER_CLIENT_PORT), "client:read") chunk_index = offset / settings.CHUNK_SIZE req = msg.ReadReq(fname, chunk_index, len) for chunk_info in msg.handle_req_response(master_comm, req, log): if chunk_info: break yield None if not chunk_info: return # pick a chunkserver to talk to # log("chunk_info: %s" % cPickle.dumps(chunk_info)) random.shuffle(chunk_info.servers) chunkaddr, port = chunk_info.servers[0] log("picked server (%s,%i)" % (chunkaddr, port)) chunk_comm = net.PakComm((chunkaddr, port), "client:read") log("sending read req") read_req = msg.ReadChunk(chunk_info.id, offset, len) for read_res in msg.handle_req_response(chunk_comm, read_req, log): if read_res: break yield None if not read_res: log("lost connectiont to chunkserver") return yield read_res
def read(fname, offset, len): """involves these steps: 1 request (fname/offset) from the master 1a master puts read lock on fname 2 receive a chunk handle and set of chunkservers with chunk 3 pick a chunkserver (nearest), request the data 4 close handle on the master 4a master releases lock """ def log(str): _log("[client:read] " + str) log("read(%s,%i,%i), connecting to (%s,%i)" % (fname, offset, len, settings.MASTER_ADDR, settings.MASTER_CLIENT_PORT)) log("connecting to master") master_comm = net.PakComm( (settings.MASTER_ADDR, settings.MASTER_CLIENT_PORT), "client:read") chunk_index = offset / settings.CHUNK_SIZE req = msg.ReadReq(fname, chunk_index, len) for chunk_info in msg.handle_req_response(master_comm, req, log): if chunk_info: break yield None if not chunk_info: return # pick a chunkserver to talk to #log("chunk_info: %s" % cPickle.dumps(chunk_info)) random.shuffle(chunk_info.servers) chunkaddr, port = chunk_info.servers[0] log("picked server (%s,%i)" % (chunkaddr, port)) chunk_comm = net.PakComm((chunkaddr, port), "client:read") log("sending read req") read_req = msg.ReadChunk(chunk_info.id, offset, len) for read_res in msg.handle_req_response(chunk_comm, read_req, log): if read_res: break yield None if not read_res: log("lost connectiont to chunkserver") return yield read_res
def file_info(fname): log("file_info(%s), connecting to (%s,%i)" % (fname, settings.MASTER_ADDR, settings.MASTER_CLIENT_PORT)) log("connecting to master") master_comm = net.PakComm((settings.MASTER_ADDR, settings.MASTER_CLIENT_PORT), "client:read") req = msg.FileInfoReq(fname) for file_info in msg.handle_req_response(master_comm, req, log): if file_info: break yield None yield file_info
def file_info(fname): log("file_info(%s), connecting to (%s,%i)" % (fname, settings.MASTER_ADDR, settings.MASTER_CLIENT_PORT)) log("connecting to master") master_comm = net.PakComm( (settings.MASTER_ADDR, settings.MASTER_CLIENT_PORT), "client:read") req = msg.FileInfoReq(fname) for file_info in msg.handle_req_response(master_comm, req, log): if file_info: break yield None yield file_info