示例#1
0
def decodeFile(setting, metadata, mode):
    '''Decode chunks into original file.'''
    n = metadata.totalnode
    k = n - 2
    nativeBlockNum = fmsrutil.getNativeBlockNum(n, k)
    parityBlockNum = fmsrutil.getParityBlockNum(n, k)

    if setting.healthynode >= setting.totalnode-2:
        dest = setting.mirrordir + '/' + metadata.filename
        src = []
        blocknums = []
        failedNodes = []
        repairChunks = []
        #Read retrieved chunks:
        for i in range(metadata.totalchunk):
            nodeid = metadata.chunkInfo[i].nodeid
            chunkname = setting.chunkdir + '/' + metadata.filename + '.chunk' + str(i)
            chunktype = metadata.chunkInfo[i].chunktype
            if metadata.chunkInfo[i].action == 'download':
                src.append(chunkname)
                blocknums.append(i)
            elif metadata.chunkInfo[i].action == 'sos':
                repairChunks.append(chunkname)
                #if nodeid not in failedNodes:
                #    failedNodes.append(nodeid)
        for i in range(metadata.totalnode):
            if setting.nodeInfo[i].healthy == False:
                failedNodes.append(i)
        destChunks = []
        if mode == 'download':
            #Decode chunks into file:
            fmsrutil.decode(n, k, src, blocknums, metadata.parityCoeff, \
                            dest, metadata.filesize, setting)
            return True
        elif mode == 'repair':
            if setting.healthynode == setting.totalnode-1:
                #Repair failed chunks by functional repair:
                #Only support 1-node recovery now.
                fmsrutil.functionalRepair(n, k, src, blocknums, failedNodes[0], \
                                          metadata.parityCoeff, repairChunks, \
                                          setting, metadata)
                return True
            else:
                return False
        else:
            return False
    else:
        return False
示例#2
0
def encodeFile(setting, metadata):
    '''Encode file into chunks.'''
    metadata.datanode = setting.datanode
    src = setting.mirrordir + '/' + metadata.filename

    n = metadata.totalnode
    k = n - 2
    nativeBlockNum = fmsrutil.getNativeBlockNum(n, k)
    parityBlockNum = fmsrutil.getParityBlockNum(n, k)
    parityCoeff = []
    path = setting.coeffdir + '/' + setting.uuid
    if not os.path.exists(path):
      parityCoeff = fmsrutil.getParityCoeff(n, k)
    else:
      with open(path, 'rb') as f:
        for i in f.read()[2:]:
          parityCoeff.append(GF256int(ord(i)))

    #Encode src file into parity chunks:
    fmsrutil.encode(n, k, src, parityCoeff, setting, metadata)
示例#3
0
def updateMetadataForDecode(setting, metadata, mode):
    '''Update metadata for retrieving chunks for decode.'''
    if setting.healthynode >= setting.totalnode - 2:
        #Case of one-node failure:
        n = metadata.totalnode
        k = n - 2
        nativeBlockNum = fmsrutil.getNativeBlockNum(n, k)
        requiredBlockNum = 0
        if mode == 'download':
            requiredBlockNum = nativeBlockNum
        elif mode == 'repair' and setting.healthynode == setting.totalnode-1:
            requiredBlockNum = metadata.totalnode - 1
        elif mode == 'repair' and setting.healthynode == setting.totalnode-2:
            #Repair of 2-node failure is not supported.
            return False
        else:
            return False

        count = 0
        if mode == 'download':
            for i in range(metadata.totalchunk):
                nodeid = metadata.chunkInfo[i].nodeid
                if setting.nodeInfo[nodeid].healthy == True:
                    metadata.chunkInfo[i].action = 'download'
                    count += 1
                    if count == requiredBlockNum:
                        break
        elif mode == 'repair':
            #Get parity coefficients:
            parityCoeff = metadata.parityCoeff[:]
            #Get the first failed node:
            failedNode = 0
            for i in range(metadata.totalnode):
                if setting.nodeInfo[i].healthy == False:
                    failedNode = i
                    break
            #Get coding information:
            nativeBlockNum = fmsrutil.getNativeBlockNum(n, k)
            parityBlockNum = fmsrutil.getParityBlockNum(n, k)
            checkNum = fmsrutil.getCheckNum(parityBlockNum)

            #Initialize strongMDSPropertyDegrees:
            strongMDSPropertyDegrees = []
            MDSproperty = False
            for i in range(n):
                strongMDSPropertyDegrees.append(0)

            fn = setting.coeffdir + '/' + setting.uuid + '.' + str(failedNode)
            if os.path.exists(fn):
              #Load enc_matrix and repairCodingCoeff directly from file (if it exists)
              print >> sys.stderr, "Using offline coefficent generation."
              with open(fn) as f:
                content = f.read()
                enc_matrix_len = ord(content[0]) + (ord(content[1]) << 8)
                enc_matrix = [[ord(content[2+row*nativeBlockNum+col]) for col in range(nativeBlockNum)] for row in range(parityBlockNum)]
                s = enc_matrix_len + 2
                num_chunks_download = ord(content[s])
                s += 1
                for chunk in metadata.chunkInfo:
                  if chunk.action == 'download':
                    chunk.action = ''
                for i in xrange(s, s+num_chunks_download):
                  metadata.chunkInfo[ord(content[i])].action = 'download'
                s += num_chunks_download
                num_chunks_repair = ord(content[s])
                cur = s + 1
                repairCodingCoeff = []
                for i in xrange(num_chunks_repair):
                  num_coeff = ord(content[cur])
                  cur += 1
                  tempCoeff = [ord(content[cur+j]) for j in xrange(num_coeff)]
                  repairCodingCoeff.append(tempCoeff)
                  cur += num_coeff
                metadata.enc_matrix = enc_matrix
                metadata.repairCodingCoeff = repairCodingCoeff
            else:
              #Loop of checking until suitable chunks are found:
              while MDSproperty == False or \
                      fmsrutil.testStrongMDSProperty(strongMDSPropertyDegrees, checkNum,n) == False:
                  blocknums = []
                  for chunk in metadata.chunkInfo:
                      if chunk.action == 'download':
                          chunk.action = ''
                  count = 0
                  for i in range(0, metadata.totalchunk, 2):
                      selectChunkInt = random.randint(0,1)
                      j = i + selectChunkInt
                      #Randomly select a chunk of each node:
                      nodeid = j / 2
                      if setting.nodeInfo[nodeid].healthy == True:
                          blocknums.append(j)
                          metadata.chunkInfo[j].action = 'download'
                          count += 1
                          if count == requiredBlockNum:
                              break

                  #Generate the encoding matrix:
                  enc_matrix = [[GF256int(0) for col in range(nativeBlockNum)] for row in range(parityBlockNum)]
                  counter = 0
                  for i in range(parityBlockNum):
                      for j in range(nativeBlockNum):
                          enc_matrix[i][j] = GF256int(parityCoeff[counter])
                          counter += 1

                  #for fmsr(k=n-2) only
                  repairCodingCoeff = [[GF256int(0) for col in range(n-1)] for row in range(n-k)]
                  for i in range(n-k):
                      for j in range(n-1):
                          repairCodingCoeff[i][j] = GF256int(random.randint(0,255))

                  #Update the encoding matrix:
                  #for fmsr(k=n-2) only
                  for i in range(nativeBlockNum):
                      for p in range(n-k):
                          enc_matrix[failedNode*2+p][i] = 0
                          for q in range(n-1):
                              enc_matrix[failedNode*2+p][i] \
                                  += repairCodingCoeff[p][q] * enc_matrix[blocknums[q]][i]

                  #Get the strongMDS property degree:
                  strongMDSPropertyDegrees = fmsrutil.checkstongMDS(n,k,nativeBlockNum,parityBlockNum,enc_matrix)
                  #Get the MDS property value:
                  MDSproperty = fmsrutil.checkMDS(n, k, enc_matrix)

                  #Store matrix for repair use
                  metadata.enc_matrix = enc_matrix
                  metadata.repairCodingCoeff = repairCodingCoeff

        if count == requiredBlockNum:
            return True
        else:
            return False
    else:
        return False