def get(self, sdfsfilename, filename): """ fetch sdfs file to local 1. Send GET message to Master to get node to get from 2. Fetch file from that node """ msg = self.constructClientFileRequestMessage("", sdfsfilename, GET) s = TCPSocket() s.connect((self.nnIp, REQUESTHANDLERPORT)) msg_serialized = msg.SerializeToString() s.sendPacket(msg_serialized) responseData = s.recvPacket() response = filerequests_pb2.FileRequest() try: response.ParseFromString(responseData) except: print("error when parsing response message") s.close() assert response.fileRequestType == filerequests_pb2.FileRequest.FILEGET msg = self.constructFileGetMsg(sdfsfilename) serialized_msg = msg.SerializeToString() localfiles = [] d = {} if len(response.members) == 0: print("No such file on sdfs") return for member in response.members: try: ss = TCPSocket() ss.connect((member.memberIp, REQUESTHANDLERPORT)) ss.sendPacket(serialized_msg) data = ss.recvPacket() new_response = filerequests_pb2.FileRequest() new_response.ParseFromString(data) thelocalfile = new_response.localfilename.split(' ')[-1] localfiles.append(thelocalfile) d[thelocalfile] = member except: print("error when asking for localfilename from datanode") print("localfiles:") print(localfiles) localfiles = [x for x in localfiles if x != ""] if len(localfiles) == 0: return timestampedfilename = sorted(localfiles, key=lambda x: x.split('_')[-1])[-1] print("timestampedfilename: %s" % timestampedfilename) member = d[localfiles[-1]] subprocess.call([ "scp", "-i", idRSAPath, userId + '@' + member.memberIp + ":" + cloudpath + timestampedfilename, filename ])
def get_versions(self, sdfsfilename, num_versions, localfilename): """ get all the last num_versions versions of the file to localfilename """ msg = self.constructClientFileRequestMessage("", sdfsfilename, GET) s = TCPSocket() s.connect((self.nnIp, REQUESTHANDLERPORT)) msg_serialized = msg.SerializeToString() s.sendPacket(msg_serialized) responseData = s.recvPacket() response = filerequests_pb2.FileRequest() try: response.ParseFromString(responseData) except: print("error when parsing response message") s.close() assert response.fileRequestType == filerequests_pb2.FileRequest.FILEGET msg = self.constructFileGetMsg(sdfsfilename) msg.clientNumVersions = num_versions serialized_msg = msg.SerializeToString() localfiles = [] d = {} print(response.members) if len(response.members) == 0: print("No such file on sdfs") return for member in response.members: try: ss = TCPSocket() ss.connect((member.memberIp, REQUESTHANDLERPORT)) ss.sendPacket(serialized_msg) data = ss.recvPacket() new_response = filerequests_pb2.FileRequest() new_response.ParseFromString(data) localfiles += new_response.localfilename.split(' ') for fn in new_response.localfilename.split(' '): d[fn] = member except: print("error when asking for localfilename from datanode") print(localfiles) localfiles = [x for x in localfiles if x != ""] if len(localfiles) == 0: return timestampedfilenames = sorted( localfiles, key=lambda x: x.split('_')[-1])[-num_versions:] member = d[localfiles[-1]] for timestampedfilename in timestampedfilenames: filename = localfilename + '_' + timestampedfilename.split('_')[-1] subprocess.call([ "scp", "-i", idRSAPath, userId + '@' + member.memberIp + ":" + cloudpath + timestampedfilename, filename ])
def constructFileGetMsg(self, sdfsfilename): msg = filerequests_pb2.FileRequest() msg.fileRequestType = filerequests_pb2.FileRequest.FILEGET msg.localfilename = "" msg.sdfsfilename = sdfsfilename msg.clientNumVersions = 1 return msg
def delete(self, sdfsfilename): """ delete sdfs file 1. Send DELETE message to Master to get nodes storing the file 2. Send FILEDELETE to nodes to delete the file """ #Sends message to Master Node that file must be deleted, the rest is handled by master Request Handler msg = self.constructClientFileRequestMessage("", sdfsfilename, DELETE) s = TCPSocket() s.connect((self.nnIp, REQUESTHANDLERPORT)) msg_serialized = msg.SerializeToString() s.sendPacket(msg_serialized) responseData = s.recvPacket() response = filerequests_pb2.FileRequest() try: response.ParseFromString(responseData) except: print("error when parsing response message") s.close() msg = self.constructFileDeleteMsg(sdfsfilename) serialized_msg = msg.SerializeToString() for member in response.members: ss = TCPSocket() ss.connect((member.memberIp, REQUESTHANDLERPORT)) ss.sendPacket(serialized_msg) ss.close()
def start(self): self.sdfsSock.bind((self.ip, REQUESTHANDLERPORT)) self.sdfsSock.listen(5) while True: msgRecvd = filerequests_pb2.FileRequest() conn, their_addr = self.sdfsSock.accept() data = conn.recvPacket() try: msgRecvd.ParseFromString(data) except: print("Error parsing from the string") # print(msgRecvd) if msgRecvd.fileRequestType == filerequests_pb2.FileRequest.PUT: self.handlePut(msgRecvd, conn) elif msgRecvd.fileRequestType == filerequests_pb2.FileRequest.METADATAUPDATE: self.handleMetadataUpdate(msgRecvd, conn) elif msgRecvd.fileRequestType == filerequests_pb2.FileRequest.FILEPUT: self.handleFilePut(msgRecvd, conn) elif msgRecvd.fileRequestType == filerequests_pb2.FileRequest.GET: self.handleGet(msgRecvd, conn) elif msgRecvd.fileRequestType == filerequests_pb2.FileRequest.FILEGET: self.handleFileGet(msgRecvd, conn) elif msgRecvd.fileRequestType == filerequests_pb2.FileRequest.LS: self.handleLs(msgRecvd, conn) elif msgRecvd.fileRequestType == filerequests_pb2.FileRequest.DELETE: self.handleDelete(msgRecvd, conn) elif msgRecvd.fileRequestType == filerequests_pb2.FileRequest.FILEDELETE: self.handleFileDelete(msgRecvd, conn) elif msgRecvd.fileRequestType == filerequests_pb2.FileRequest.REREPLICATION: self.handleRereplication(msgRecvd, conn) elif msgRecvd.fileRequestType == filerequests_pb2.FileRequest.UPDATECLOUDFILES: self.handleUpdateCloudFiles(msgRecvd, conn)
def constructRereplicationMsg(self, sdfsfilename, candidate_nodes): msg = filerequests_pb2.FileRequest() msg.fileRequestType = filerequests_pb2.FileRequest.REREPLICATION msg.localfilename = "" msg.sdfsfilename = sdfsfilename for member in candidate_nodes: addedmember = msg.members.add() addedmember.memberId = member.id addedmember.memberIp = member.ip addedmember.memberPort = member.port return msg
def constructUpdateCloudFilesMsg(self, sdfsfilename, nodes): msg = filerequests_pb2.FileRequest() msg.fileRequestType = filerequests_pb2.FileRequest.UPDATECLOUDFILES msg.localfilename = "" msg.sdfsfilename = sdfsfilename for member in nodes: addedmember = msg.members.add() addedmember.memberId = member.id addedmember.memberIp = member.ip addedmember.memberPort = member.port return msg
def put(self, filename, sdfsfilename): """ upload local file to sdfs 1. Send a PUT message to Master to get the nodes to put on 2. transfer the file to each node along with a FILEPUT message """ #Check if file exists in the first place if (not os.path.exists(filename)): print("File could not be found on local directory") return #In the case of an absolute file path, have to ignore the directories and get the last value present(actual filename) fileSplit = filename.split("/") truefilename = fileSplit[len(fileSplit) - 1] #First off, construct a fileRequest of type PUT to know which of the 4 VMs it should transfer the file to msg = self.constructClientFileRequestMessage(truefilename, sdfsfilename, PUT) #Send off the message to the master node and then wait for a response s = TCPSocket() s.connect((self.nnIp, REQUESTHANDLERPORT)) msg_serialized = msg.SerializeToString() s.sendPacket(msg_serialized) responseData = s.recvPacket() response = filerequests_pb2.FileRequest() try: response.ParseFromString(responseData) except: print("error when parsing response message") s.close() assert response.fileRequestType == filerequests_pb2.FileRequest.FILEPUT timestamp = datetime.datetime.now().isoformat() timestampedfilename = response.sdfsfilename + "_" + timestamp msg = self.constructFilePutMsg(timestampedfilename, response.sdfsfilename, response.members) serialized_msg = msg.SerializeToString() for member in response.members: try: subprocess.call([ "scp", "-i", idRSAPath, filename, userId + '@' + member.memberIp + ":" + cloudpath + timestampedfilename ]) ss = TCPSocket() ss.connect((member.memberIp, REQUESTHANDLERPORT)) ss.sendPacket(serialized_msg) ss.close() except: print("error when transferring file to data node") s = TCPSocket() s.connect((self.nnIp, REQUESTHANDLERPORT)) response.fileRequestType = filerequests_pb2.FileRequest.METADATAUPDATE s.sendPacket(response.SerializeToString()) s.close()
def constructFilePutMsg(self, localfilename, sdfsfilename, members): msg = filerequests_pb2.FileRequest() msg.fileRequestType = filerequests_pb2.FileRequest.FILEPUT msg.localfilename = localfilename msg.sdfsfilename = sdfsfilename for member in members: addedmember = msg.members.add() addedmember.memberId = member.memberId addedmember.memberIp = member.memberIp addedmember.memberPort = member.memberPort return msg
def constructFileGetMsg(self, sdfsfilename, listofmembers): msg = filerequests_pb2.FileRequest() msg.fileRequestType = filerequests_pb2.FileRequest.FILEGET msg.localfilename = "" msg.sdfsfilename = sdfsfilename for member in listofmembers: addedmember = msg.members.add() addedmember.memberId = member.id addedmember.memberIp = member.ip addedmember.memberPort = member.port return msg
def constructClientFileRequestMessage(self, localfilename, sdfsfilename, fileRequestType): msg = filerequests_pb2.FileRequest() msg.localfilename = localfilename msg.sdfsfilename = sdfsfilename if (fileRequestType == GET): msg.fileRequestType = filerequests_pb2.FileRequest.GET elif (fileRequestType == PUT): msg.fileRequestType = filerequests_pb2.FileRequest.PUT elif (fileRequestType == DELETE): msg.fileRequestType = filerequests_pb2.FileRequest.DELETE elif (fileRequestType == LS): msg.fileRequestType = filerequests_pb2.FileRequest.LS elif (filerequests_pb2 == NV): msg.fileRequestType = filerequests_pb2.FileRequest.GETVERSIONS return msg
def ls(self, sdfsfilename): """ list all node where this file is currently stored on sdfs Lookup Master's metadata """ msg = self.constructClientFileRequestMessage("", sdfsfilename, LS) s = TCPSocket() s.connect((self.nnIp, REQUESTHANDLERPORT)) msg_serialized = msg.SerializeToString() s.sendPacket(msg_serialized) responseData = s.recvPacket() response = filerequests_pb2.FileRequest() try: response.ParseFromString(responseData) except: print("error when parsing response message") s.close() if len(response.members) == 0: print("No such file on sdfs") return for member in response.members: print(member.memberId)
def constructFileDeleteMsg(self, sdfsfilename): msg = filerequests_pb2.FileRequest() msg.fileRequestType = filerequests_pb2.FileRequest.FILEDELETE msg.localfilename = "" msg.sdfsfilename = sdfsfilename return msg
def constructGetMsg(self, localfilename): msg = filerequests_pb2.FileRequest() msg.fileRequestType = filerequests_pb2.FileRequest.GET msg.localfilename = localfilename msg.sdfsfilename = "" return msg