def upfile(stub, sourcepath, destination):
contentarray ,sizes= chunk.split(sourcepath)
response = stub.getChunkInfoAndAllocatedDataServer(MasterForClient_pb2.getChunkInfoAndAllocatedDataServerRequest(
size = sizes,
path = destination
))
chunknum = 0
address = []
allchunk = []
for feature in response:
chunks = chunk.chunk()
chunks.ChunkSize = feature.ChunkSize
chunks.setCID(feature.ChunkId)
chunks.setFileInfo(feature.inFID,feature.offset)
chunks.setDID(feature.StoreDID)
ip = feature.ip
port = feature.port
chunks.setContent(contentarray[chunknum])
chunknum += 1
address.append(str(ip)+':'+str(port))
allchunk.append(chunks)
""" with multiprocessing.Pool(4) as p:
results = p.map(SendChunkToDataserver, zip(address,allchunk)) """
results = []
for i in range(len(allchunk)):
results.append(SendChunkToDataserver((address[i],allchunk[i])))
print('upload ok')
print(results)
def downloadFile(stub, cur_node, user_input):
path = user_input.split()[1]
path = cur_node.path + '/' + path
if filetree.FileTree.isUniquePath(path):
print(colored('No such file.', 'red'))
return -1
chunksList = requestDownloadFromMaster(stub, path)
dataList = []
for chk in chunksList:
if chk.status == 0:
print('Houston We Have a Problem --\nSomething Goes Wrong!')
return 1
ip = chk.ip
port = chk.port
cid = chk.ChunkId
mystub = ConnectDataServer(ip + ':' + str(port))
chunkData = downloadChunk(mystub, cid)
theChunk = chunk.chunk()
theChunk.ChunkSize = chunkData.ChunkSize
theChunk.ChunkId = chunkData.ChunkId
theChunk.inFID = chunkData.inFID
theChunk.offset = chunkData.offset
theChunk.Content = chunkData.Content
dataList.append(theChunk)
name = user_input.split()[1]
if len(user_input.split()) > 2:
name = user_input.split()[2]
chunk.merge(dataList, name)
print('%s Download Successful!!'% name)
return 0
def copyChunk(self, request, context):
cchunk = chunk.chunk()
metadata = request.metadata
content = request.chunk
cchunk.ChunkSize = metadata.ChunkSize
cchunk.ChunkId = metadata.ChunkId
cchunk.inFID = metadata.inFID
cchunk.offset = metadata.offset
cchunk.StoreDID = StoreManager.StoreManager.getDID()
cchunk.Content = content
StoreManager.StoreManager.store(cchunk, True)
print(cchunk.ChunkId, 'Backup Done!')
return DataForClient_pb2.copyChunkResponse(Msg='ok')
def uploadChunk(self, request, context):
cchunk = chunk.chunk()
metadata = request.metadata
content = request.chunk
cchunk.ChunkSize = metadata.ChunkSize
cchunk.ChunkId = metadata.ChunkId
cchunk.inFID = metadata.inFID
cchunk.offset = metadata.offset
cchunk.StoreDID = metadata.StoreDID
cchunk.Content = content
print(cchunk.ChunkId, 'Done!')
StoreManager.StoreManager.store(cchunk)
# 投票
vote(cchunk.getFileID(), cchunk.getChunkId(), 1)
return DataForClient_pb2.uploadChunkResponse(Msg='Saved!')
def startbackup():
while True:
if not FileManager.sys.sizeOfonlineDataserver() > 1:
print('The number of DataServer <= 1,The Backup sys close')
break
task = Backup.BackupManager.start()
if task is None:
break
fid = task[0]
cid = task[1]
isCreatTask = task[2]
if isCreatTask:
achunk = FileManager.sys.getChunk(fid, cid)
if achunk is None:
continue
did = achunk.getDataserverID()
if Backup.BackupManager.isexist(cid, did):
continue
newchunk = chunk.chunk()
newcid = FileManager.sys.getNewCID()
newdid = FileManager.sys.FindBackupServer(achunk.getDataserverID())
newchunk.setCID(newcid)
newchunk.setDID(newdid)
newchunk.setFileInfo(achunk.getFileID(), achunk.getOffset())
newip, newport = FileManager.sys.SeekSocket(newdid)
try:
stub = ConnectDataServer(did)
# send infomation to dataserver(ip,port) copy cid-chunk to newdataserver(newip,newport) as newcid-chunk
copyChunkBetweenDataServer(stub, cid, newip, newport, newcid)
FileManager.sys.upchunkonRegister(newdid, 1, newchunk)
Backup.BackupManager.end(cid, newchunk)
except:
print(str(cid) + 'create backup failed!')
else:
bq = Backup.BackupManager.getAbackupQue(cid)
if not bq is None:
for achunk in bq.getall():
adid = achunk.getDataserverID()
try:
stub = ConnectDataServer(adid)
deleteChunkOnDataServer(stub, achunk.getChunkId())
FileManager.sys.upchunkonRegister(adid, -1, achunk)
except:
print(
str(achunk.getChunkID()) + 'delete backup failed!')
Backup.BackupManager.end(cid)
def CreateFile(self, path, size):
newFile = AFile()
newFile.setpath(path)
self.FIDcout += 1
newFile.setFID(self.FIDcout)
newFile.setFszie(size)
chunknumber = math.ceil(size/chunk.CHUNK_SIZE)
for i in range(chunknumber):
newChunk = chunk.chunk()
self.CIDcout += 1
newChunk.setCID(self.CIDcout)
newChunk.setFileInfo(newFile.getFID(),i)
newChunk.setDID(self.FindDataServer())
self.upchunkonRegister(newChunk.getDataserverID(),1,newChunk)
newFile.appendChunk(newChunk)
self.FileSystem.setdefault(newFile.getFID(),newFile)
self.show()
return newFile
print(
' ChunkID | from(FileID,offset) |store(Address)| ChunkSize | status'
)
for key, achunk in self.UsedChunk.items():
print('No.%-10d %5d %5d %20d %15d %20s' %
(key, achunk.getFileID(), achunk.getOffset(),
achunk.getDataserverID(), achunk.ChunkSize,
'Permanent Storage'))
for key, achunk in self.TmpChunk.items():
print('No.%-10d %5d %5d %20d %15d %20s' %
(key, achunk.getFileID(), achunk.getOffset(),
achunk.getDataserverID(), achunk.ChunkSize,
'Temporary Storage'))
StoreManager = StoreManage()
if __name__ == '__main__':
StoreManager.setDID(987)
achunk = chunk.chunk()
achunk.setCID(7)
achunk.setContent('are you OK?')
StoreManager.store(achunk)
StoreManager.commit(7)
StoreManager.aborted(7)
bchunk = StoreManager.get(7)
if bchunk is not None:
print(bchunk.getContent())
else:
print('NONE')
self.createMainchunk(maincid)
self.createAbackup(maincid, achunk)
self.show()
def show(self):
print(
'------------------------BackupManager Table----------------------------------------'
)
print(
' BackupChunkID | mainChunkID |store(DataServerID)| ChunkSize'
)
# print('---------------------------------------------------------------------------------')
for maincid, record in self.MainAndBackupList.items():
for abackup in record.getall():
print('No.%-10d %10d %20d %20d' %
(abackup.getChunkId(), maincid,
abackup.getDataserverID(), abackup.ChunkSize))
BackupManager = BackupManage()
if __name__ == '__main__':
ac = chunk.chunk()
ac.setCID(666)
bc = chunk.chunk()
bc.setCID(444)
BackupManager.createMainchunk(9)
BackupManager.createAbackup(9, ac)
BackupManager.createAbackup(9, bc)
newmainchunk = BackupManager.updateMainchunk(9)
print(BackupManager.getAbackup(newmainchunk.getChunkId()).getChunkId())