def encodeFile(setting, metadata):
'''Encode a file into chunks.'''
metadata.datanode = setting.datanode
src = setting.mirrordir + '/' + metadata.filename
dest = []
dataChunks = []
for i in range(metadata.datanode):
chunkname = setting.chunkdir + '/' + metadata.filename + '.node' + str(
i)
dataChunks.append(chunkname)
dest.append(chunkname)
parityChunk = []
for i in range(metadata.datanode, metadata.totalnode):
parityname = setting.chunkdir + '/' + metadata.filename + '.node' + str(
i)
parityChunk.append(parityname)
dest.append(parityname)
codingutil.ecEncode(src, dataChunks, parityChunk, setting)
for i in range(metadata.totalnode):
chunk = common.ChunkMetadata(i)
chunk.chunkname = metadata.filename + '.node' + str(i)
if i >= metadata.datanode:
chunk.chunktype = 'parity'
else:
chunk.chunktype = 'native'
chunk.chunkpath = dest[i]
chunk.chunksize = setting.chunksize
chunk.nodeid = i
chunk.nodekey = setting.nodeInfo[i].nodekey
chunk.nodetype = setting.nodeInfo[i].nodetype
chunk.bucketname = setting.nodeInfo[i].bucketname
chunk.action = 'upload'
chunk.position = 0 #Position inside big-chunk
metadata.chunkInfo.append(chunk)
metadata.totalchunk = metadata.totalnode
#Add support for big-chunk:
for i in range(metadata.totalnode):
fileNode = common.FileNodeMetadata(i)
fileNode.nodekey = setting.nodeInfo[i].nodekey
fileNode.nodetype = setting.nodeInfo[i].nodetype
fileNode.bucketname = setting.nodeInfo[i].bucketname
fileNode.bigchunksize = metadata.chunkInfo[i].chunksize
fileNode.bigchunkpath = metadata.chunkInfo[i].chunkpath
fileNode.bigchunkname = metadata.chunkInfo[i].chunkname
fileNode.chunknum = 1
fileNode.action = 'upload'
metadata.fileNodeInfo.append(fileNode)
def encodeFile(setting, metadata):
'''Encode a file into chunks.'''
metadata.datanode = setting.datanode
src = setting.mirrordir + '/' + metadata.filename
dest = []
dataChunks = []
for i in range(metadata.datanode):
chunkname = setting.chunkdir + '/' + metadata.filename + '.node' + str(i)
dataChunks.append(chunkname)
dest.append(chunkname)
parityChunk = []
for i in range(metadata.datanode, metadata.totalnode):
parityname = setting.chunkdir + '/' + metadata.filename + '.node' + str(i)
parityChunk.append(parityname)
dest.append(parityname)
codingutil.ecEncode(src, dataChunks, parityChunk, setting)
for i in range(metadata.totalnode):
chunk = common.ChunkMetadata(i)
chunk.chunkname = metadata.filename + '.node' + str(i)
if i >= metadata.datanode:
chunk.chunktype = 'parity'
else:
chunk.chunktype = 'native'
chunk.chunkpath = dest[i]
chunk.chunksize = setting.chunksize
chunk.nodeid = i
chunk.nodekey = setting.nodeInfo[i].nodekey
chunk.nodetype = setting.nodeInfo[i].nodetype
chunk.bucketname = setting.nodeInfo[i].bucketname
chunk.action = 'upload'
chunk.position = 0 #Position inside big-chunk
metadata.chunkInfo.append(chunk)
metadata.totalchunk = metadata.totalnode
#Add support for big-chunk:
for i in range(metadata.totalnode):
fileNode = common.FileNodeMetadata(i)
fileNode.nodekey = setting.nodeInfo[i].nodekey
fileNode.nodetype = setting.nodeInfo[i].nodetype
fileNode.bucketname = setting.nodeInfo[i].bucketname
fileNode.bigchunksize = metadata.chunkInfo[i].chunksize
fileNode.bigchunkpath = metadata.chunkInfo[i].chunkpath
fileNode.bigchunkname = metadata.chunkInfo[i].chunkname
fileNode.chunknum = 1
fileNode.action = 'upload'
metadata.fileNodeInfo.append(fileNode)
def encodeFile(setting, metadata):
'''Encode file into chunks.'''
metadata.datanode = setting.datanode
src = setting.mirrordir + '/' + metadata.filename
dest = []
dataChunks = []
parityChunks = []
n = metadata.totalnode
k = metadata.datanode
nativeBlockNum = embrutil.getNativeBlockNum(n, k)
parityBlockNum = embrutil.getParityBlockNum(n, k)
authenticNum = nativeBlockNum + parityBlockNum
nodeIdList = embrutil.getNodeIdList(n, k)
for i in range(nativeBlockNum):
chunkname = setting.chunkdir + '/' + metadata.filename + '.chunk' + str(
i)
dataChunks.append(chunkname)
dest.append(chunkname)
for i in range(parityBlockNum):
parityname = setting.chunkdir + '/'+ metadata.filename + '.chunk' \
+ str(authenticNum-parityBlockNum+i)
parityChunks.append(parityname)
dest.append(parityname)
codingutil.ecEncode(src, dataChunks, parityChunks, setting)
#Generate info for big-chunk:
bigChunkPaths = []
for i in range(metadata.totalnode):
fileNode = common.FileNodeMetadata(i)
fileNode.nodekey = setting.nodeInfo[i].nodekey
fileNode.nodetype = setting.nodeInfo[i].nodetype
fileNode.bucketname = setting.nodeInfo[i].bucketname
fileNode.bigchunksize = 0
fileNode.chunknum = 0
metadata.fileNodeInfo.append(fileNode)
bigChunkPaths.append([])
#Generate info for small chunks:
for i in range(authenticNum * 2):
chunk = common.ChunkMetadata(i)
if i < authenticNum:
#Case of non-replica:
chunk.chunkname = metadata.filename + '.chunk' + str(i)
chunk.chunksize = os.path.getsize(dest[i])
if i < nativeBlockNum:
chunk.chunktype = 'native'
else:
chunk.chunktype = 'parity'
chunk.chunkpath = dest[i]
else:
#Case of replica:
j = i - authenticNum
chunk.chunkname = metadata.chunkInfo[j].chunkname
chunk.chunksize = metadata.chunkInfo[j].chunksize
chunk.chunktype = 'replica'
chunk.chunkpath = metadata.chunkInfo[j].chunkpath
nodeid = nodeIdList[i]
chunk.nodeid = nodeid
chunk.nodekey = setting.nodeInfo[nodeid].nodekey
chunk.nodetype = setting.nodeInfo[nodeid].nodetype
chunk.bucketname = setting.nodeInfo[nodeid].bucketname
chunk.action = 'upload'
#Add chunk position inside big-chunk:
chunk.position = metadata.fileNodeInfo[nodeid].chunknum
metadata.chunkInfo.append(chunk)
#Add support for big-chunk:
metadata.fileNodeInfo[nodeid].bigchunksize += chunk.chunksize
metadata.fileNodeInfo[nodeid].chunknum += 1
bigChunkPaths[nodeid].append(chunk.chunkpath)
metadata.totalchunk = authenticNum * 2
#Generate big-chunks
for i in range(metadata.totalnode):
dest = setting.chunkdir + '/' + metadata.filename + '.node' + str(i)
codingutil.join(bigChunkPaths[i], dest,
metadata.fileNodeInfo[i].bigchunksize)
metadata.fileNodeInfo[i].bigchunkpath = dest
metadata.fileNodeInfo[
i].bigchunkname = metadata.filename + '.node' + str(i)
metadata.fileNodeInfo[i].action = 'upload'
def encodeFile(setting, metadata):
'''Encode file into chunks.'''
metadata.datanode = setting.datanode
src = setting.mirrordir + '/' + metadata.filename
dest = []
dataChunks = []
parityChunks = []
n = metadata.totalnode
k = metadata.datanode
nativeBlockNum = embrutil.getNativeBlockNum(n, k)
parityBlockNum = embrutil.getParityBlockNum(n, k)
authenticNum = nativeBlockNum + parityBlockNum
nodeIdList = embrutil.getNodeIdList(n, k)
for i in range(nativeBlockNum):
chunkname = setting.chunkdir + '/' + metadata.filename + '.chunk' + str(i)
dataChunks.append(chunkname)
dest.append(chunkname)
for i in range(parityBlockNum):
parityname = setting.chunkdir + '/'+ metadata.filename + '.chunk' \
+ str(authenticNum-parityBlockNum+i)
parityChunks.append(parityname)
dest.append(parityname)
codingutil.ecEncode(src, dataChunks, parityChunks, setting)
#Generate info for big-chunk:
bigChunkPaths = []
for i in range(metadata.totalnode):
fileNode = common.FileNodeMetadata(i)
fileNode.nodekey = setting.nodeInfo[i].nodekey
fileNode.nodetype = setting.nodeInfo[i].nodetype
fileNode.bucketname = setting.nodeInfo[i].bucketname
fileNode.bigchunksize = 0
fileNode.chunknum = 0
metadata.fileNodeInfo.append(fileNode)
bigChunkPaths.append([])
#Generate info for small chunks:
for i in range(authenticNum*2):
chunk = common.ChunkMetadata(i)
if i < authenticNum:
#Case of non-replica:
chunk.chunkname = metadata.filename + '.chunk' + str(i)
chunk.chunksize = os.path.getsize(dest[i])
if i < nativeBlockNum:
chunk.chunktype = 'native'
else:
chunk.chunktype = 'parity'
chunk.chunkpath = dest[i]
else:
#Case of replica:
j = i - authenticNum
chunk.chunkname = metadata.chunkInfo[j].chunkname
chunk.chunksize = metadata.chunkInfo[j].chunksize
chunk.chunktype = 'replica'
chunk.chunkpath = metadata.chunkInfo[j].chunkpath
nodeid = nodeIdList[i]
chunk.nodeid = nodeid
chunk.nodekey = setting.nodeInfo[nodeid].nodekey
chunk.nodetype = setting.nodeInfo[nodeid].nodetype
chunk.bucketname = setting.nodeInfo[nodeid].bucketname
chunk.action = 'upload'
#Add chunk position inside big-chunk:
chunk.position = metadata.fileNodeInfo[nodeid].chunknum
metadata.chunkInfo.append(chunk)
#Add support for big-chunk:
metadata.fileNodeInfo[nodeid].bigchunksize += chunk.chunksize
metadata.fileNodeInfo[nodeid].chunknum += 1
bigChunkPaths[nodeid].append(chunk.chunkpath)
metadata.totalchunk = authenticNum * 2
#Generate big-chunks
for i in range(metadata.totalnode):
dest = setting.chunkdir + '/' + metadata.filename + '.node' + str(i)
codingutil.join(bigChunkPaths[i],dest,metadata.fileNodeInfo[i].bigchunksize)
metadata.fileNodeInfo[i].bigchunkpath = dest
metadata.fileNodeInfo[i].bigchunkname = metadata.filename + '.node' + str(i)
metadata.fileNodeInfo[i].action = 'upload'
