def get(self, sdfsfilename, filename):
"""
fetch sdfs file to local
1. Send GET message to Master to get node to get from
2. Fetch file from that node
"""
msg = self.constructClientFileRequestMessage("", sdfsfilename, GET)
s = TCPSocket()
s.connect((self.nnIp, REQUESTHANDLERPORT))
msg_serialized = msg.SerializeToString()
s.sendPacket(msg_serialized)
responseData = s.recvPacket()
response = filerequests_pb2.FileRequest()
try:
response.ParseFromString(responseData)
except:
print("error when parsing response message")
s.close()
assert response.fileRequestType == filerequests_pb2.FileRequest.FILEGET
msg = self.constructFileGetMsg(sdfsfilename)
serialized_msg = msg.SerializeToString()
localfiles = []
d = {}
if len(response.members) == 0:
print("No such file on sdfs")
return
for member in response.members:
try:
ss = TCPSocket()
ss.connect((member.memberIp, REQUESTHANDLERPORT))
ss.sendPacket(serialized_msg)
data = ss.recvPacket()
new_response = filerequests_pb2.FileRequest()
new_response.ParseFromString(data)
thelocalfile = new_response.localfilename.split(' ')[-1]
localfiles.append(thelocalfile)
d[thelocalfile] = member
except:
print("error when asking for localfilename from datanode")
print("localfiles:")
print(localfiles)
localfiles = [x for x in localfiles if x != ""]
if len(localfiles) == 0:
return
timestampedfilename = sorted(localfiles,
key=lambda x: x.split('_')[-1])[-1]
print("timestampedfilename: %s" % timestampedfilename)
member = d[localfiles[-1]]
subprocess.call([
"scp", "-i", idRSAPath, userId + '@' + member.memberIp + ":" +
cloudpath + timestampedfilename, filename
])
def get_versions(self, sdfsfilename, num_versions, localfilename):
"""
get all the last num_versions versions of the file to localfilename
"""
msg = self.constructClientFileRequestMessage("", sdfsfilename, GET)
s = TCPSocket()
s.connect((self.nnIp, REQUESTHANDLERPORT))
msg_serialized = msg.SerializeToString()
s.sendPacket(msg_serialized)
responseData = s.recvPacket()
response = filerequests_pb2.FileRequest()
try:
response.ParseFromString(responseData)
except:
print("error when parsing response message")
s.close()
assert response.fileRequestType == filerequests_pb2.FileRequest.FILEGET
msg = self.constructFileGetMsg(sdfsfilename)
msg.clientNumVersions = num_versions
serialized_msg = msg.SerializeToString()
localfiles = []
d = {}
print(response.members)
if len(response.members) == 0:
print("No such file on sdfs")
return
for member in response.members:
try:
ss = TCPSocket()
ss.connect((member.memberIp, REQUESTHANDLERPORT))
ss.sendPacket(serialized_msg)
data = ss.recvPacket()
new_response = filerequests_pb2.FileRequest()
new_response.ParseFromString(data)
localfiles += new_response.localfilename.split(' ')
for fn in new_response.localfilename.split(' '):
d[fn] = member
except:
print("error when asking for localfilename from datanode")
print(localfiles)
localfiles = [x for x in localfiles if x != ""]
if len(localfiles) == 0:
return
timestampedfilenames = sorted(
localfiles, key=lambda x: x.split('_')[-1])[-num_versions:]
member = d[localfiles[-1]]
for timestampedfilename in timestampedfilenames:
filename = localfilename + '_' + timestampedfilename.split('_')[-1]
subprocess.call([
"scp", "-i", idRSAPath, userId + '@' + member.memberIp + ":" +
cloudpath + timestampedfilename, filename
])
def constructFileGetMsg(self, sdfsfilename):
msg = filerequests_pb2.FileRequest()
msg.fileRequestType = filerequests_pb2.FileRequest.FILEGET
msg.localfilename = ""
msg.sdfsfilename = sdfsfilename
msg.clientNumVersions = 1
return msg
def delete(self, sdfsfilename):
"""
delete sdfs file
1. Send DELETE message to Master to get nodes storing the file
2. Send FILEDELETE to nodes to delete the file
"""
#Sends message to Master Node that file must be deleted, the rest is handled by master Request Handler
msg = self.constructClientFileRequestMessage("", sdfsfilename, DELETE)
s = TCPSocket()
s.connect((self.nnIp, REQUESTHANDLERPORT))
msg_serialized = msg.SerializeToString()
s.sendPacket(msg_serialized)
responseData = s.recvPacket()
response = filerequests_pb2.FileRequest()
try:
response.ParseFromString(responseData)
except:
print("error when parsing response message")
s.close()
msg = self.constructFileDeleteMsg(sdfsfilename)
serialized_msg = msg.SerializeToString()
for member in response.members:
ss = TCPSocket()
ss.connect((member.memberIp, REQUESTHANDLERPORT))
ss.sendPacket(serialized_msg)
ss.close()
def start(self):
self.sdfsSock.bind((self.ip, REQUESTHANDLERPORT))
self.sdfsSock.listen(5)
while True:
msgRecvd = filerequests_pb2.FileRequest()
conn, their_addr = self.sdfsSock.accept()
data = conn.recvPacket()
try:
msgRecvd.ParseFromString(data)
except:
print("Error parsing from the string")
# print(msgRecvd)
if msgRecvd.fileRequestType == filerequests_pb2.FileRequest.PUT:
self.handlePut(msgRecvd, conn)
elif msgRecvd.fileRequestType == filerequests_pb2.FileRequest.METADATAUPDATE:
self.handleMetadataUpdate(msgRecvd, conn)
elif msgRecvd.fileRequestType == filerequests_pb2.FileRequest.FILEPUT:
self.handleFilePut(msgRecvd, conn)
elif msgRecvd.fileRequestType == filerequests_pb2.FileRequest.GET:
self.handleGet(msgRecvd, conn)
elif msgRecvd.fileRequestType == filerequests_pb2.FileRequest.FILEGET:
self.handleFileGet(msgRecvd, conn)
elif msgRecvd.fileRequestType == filerequests_pb2.FileRequest.LS:
self.handleLs(msgRecvd, conn)
elif msgRecvd.fileRequestType == filerequests_pb2.FileRequest.DELETE:
self.handleDelete(msgRecvd, conn)
elif msgRecvd.fileRequestType == filerequests_pb2.FileRequest.FILEDELETE:
self.handleFileDelete(msgRecvd, conn)
elif msgRecvd.fileRequestType == filerequests_pb2.FileRequest.REREPLICATION:
self.handleRereplication(msgRecvd, conn)
elif msgRecvd.fileRequestType == filerequests_pb2.FileRequest.UPDATECLOUDFILES:
self.handleUpdateCloudFiles(msgRecvd, conn)
def constructRereplicationMsg(self, sdfsfilename, candidate_nodes):
msg = filerequests_pb2.FileRequest()
msg.fileRequestType = filerequests_pb2.FileRequest.REREPLICATION
msg.localfilename = ""
msg.sdfsfilename = sdfsfilename
for member in candidate_nodes:
addedmember = msg.members.add()
addedmember.memberId = member.id
addedmember.memberIp = member.ip
addedmember.memberPort = member.port
return msg
def constructUpdateCloudFilesMsg(self, sdfsfilename, nodes):
msg = filerequests_pb2.FileRequest()
msg.fileRequestType = filerequests_pb2.FileRequest.UPDATECLOUDFILES
msg.localfilename = ""
msg.sdfsfilename = sdfsfilename
for member in nodes:
addedmember = msg.members.add()
addedmember.memberId = member.id
addedmember.memberIp = member.ip
addedmember.memberPort = member.port
return msg
def put(self, filename, sdfsfilename):
"""
upload local file to sdfs
1. Send a PUT message to Master to get the nodes to put on
2. transfer the file to each node along with a FILEPUT message
"""
#Check if file exists in the first place
if (not os.path.exists(filename)):
print("File could not be found on local directory")
return
#In the case of an absolute file path, have to ignore the directories and get the last value present(actual filename)
fileSplit = filename.split("/")
truefilename = fileSplit[len(fileSplit) - 1]
#First off, construct a fileRequest of type PUT to know which of the 4 VMs it should transfer the file to
msg = self.constructClientFileRequestMessage(truefilename,
sdfsfilename, PUT)
#Send off the message to the master node and then wait for a response
s = TCPSocket()
s.connect((self.nnIp, REQUESTHANDLERPORT))
msg_serialized = msg.SerializeToString()
s.sendPacket(msg_serialized)
responseData = s.recvPacket()
response = filerequests_pb2.FileRequest()
try:
response.ParseFromString(responseData)
except:
print("error when parsing response message")
s.close()
assert response.fileRequestType == filerequests_pb2.FileRequest.FILEPUT
timestamp = datetime.datetime.now().isoformat()
timestampedfilename = response.sdfsfilename + "_" + timestamp
msg = self.constructFilePutMsg(timestampedfilename,
response.sdfsfilename, response.members)
serialized_msg = msg.SerializeToString()
for member in response.members:
try:
subprocess.call([
"scp", "-i", idRSAPath, filename, userId + '@' +
member.memberIp + ":" + cloudpath + timestampedfilename
])
ss = TCPSocket()
ss.connect((member.memberIp, REQUESTHANDLERPORT))
ss.sendPacket(serialized_msg)
ss.close()
except:
print("error when transferring file to data node")
s = TCPSocket()
s.connect((self.nnIp, REQUESTHANDLERPORT))
response.fileRequestType = filerequests_pb2.FileRequest.METADATAUPDATE
s.sendPacket(response.SerializeToString())
s.close()
def constructFilePutMsg(self, localfilename, sdfsfilename, members):
msg = filerequests_pb2.FileRequest()
msg.fileRequestType = filerequests_pb2.FileRequest.FILEPUT
msg.localfilename = localfilename
msg.sdfsfilename = sdfsfilename
for member in members:
addedmember = msg.members.add()
addedmember.memberId = member.memberId
addedmember.memberIp = member.memberIp
addedmember.memberPort = member.memberPort
return msg
def constructFileGetMsg(self, sdfsfilename, listofmembers):
msg = filerequests_pb2.FileRequest()
msg.fileRequestType = filerequests_pb2.FileRequest.FILEGET
msg.localfilename = ""
msg.sdfsfilename = sdfsfilename
for member in listofmembers:
addedmember = msg.members.add()
addedmember.memberId = member.id
addedmember.memberIp = member.ip
addedmember.memberPort = member.port
return msg
def constructClientFileRequestMessage(self, localfilename, sdfsfilename,
fileRequestType):
msg = filerequests_pb2.FileRequest()
msg.localfilename = localfilename
msg.sdfsfilename = sdfsfilename
if (fileRequestType == GET):
msg.fileRequestType = filerequests_pb2.FileRequest.GET
elif (fileRequestType == PUT):
msg.fileRequestType = filerequests_pb2.FileRequest.PUT
elif (fileRequestType == DELETE):
msg.fileRequestType = filerequests_pb2.FileRequest.DELETE
elif (fileRequestType == LS):
msg.fileRequestType = filerequests_pb2.FileRequest.LS
elif (filerequests_pb2 == NV):
msg.fileRequestType = filerequests_pb2.FileRequest.GETVERSIONS
return msg
def ls(self, sdfsfilename):
"""
list all node where this file is currently stored on sdfs
Lookup Master's metadata
"""
msg = self.constructClientFileRequestMessage("", sdfsfilename, LS)
s = TCPSocket()
s.connect((self.nnIp, REQUESTHANDLERPORT))
msg_serialized = msg.SerializeToString()
s.sendPacket(msg_serialized)
responseData = s.recvPacket()
response = filerequests_pb2.FileRequest()
try:
response.ParseFromString(responseData)
except:
print("error when parsing response message")
s.close()
if len(response.members) == 0:
print("No such file on sdfs")
return
for member in response.members:
print(member.memberId)
def constructFileDeleteMsg(self, sdfsfilename):
msg = filerequests_pb2.FileRequest()
msg.fileRequestType = filerequests_pb2.FileRequest.FILEDELETE
msg.localfilename = ""
msg.sdfsfilename = sdfsfilename
return msg
def constructGetMsg(self, localfilename):
msg = filerequests_pb2.FileRequest()
msg.fileRequestType = filerequests_pb2.FileRequest.GET
msg.localfilename = localfilename
msg.sdfsfilename = ""
return msg
