def encode_file_not_really_and_hash(inf, cb, k, m, chunksize=4096): hasher = sha1.new() enc = zfec.Encoder(k, m) l = tuple([ array.array('c') for i in range(k) ]) indatasize = k*chunksize # will be reset to shorter upon EOF eof = False ZEROES=array.array('c', ['\x00'])*chunksize while not eof: # This loop body executes once per segment. i = 0 while (i<len(l)): # This loop body executes once per chunk. a = l[i] del a[:] try: a.fromfile(inf, chunksize) i += 1 except EOFError: eof = True indatasize = i*chunksize + len(a) # padding a.fromstring("\x00" * (chunksize-len(a))) i += 1 while (i<len(l)): a = l[i] a[:] = ZEROES i += 1 # res = enc.encode(l) for thing in l: hasher.update(thing) cb(None, None)
def ecDecodeChunks(n, k, src, blocknums, dest, setting, failed=[], nativeFailed=True): '''Generate a full list of chunks.''' '''Use zfec library.''' if nativeFailed: decoder = zfec.Decoder(k, n) indatalist = [] for filename in src: infile = open(filename, 'rb') indatalist.append(infile.read()) infile.close() #Get native data chunks: outdatalist = decoder.decode(indatalist, blocknums) else: outdatalist = [] for filename in src: infile = open(filename, 'rb') outdatalist.append(infile.read()) infile.close() #Generate full list of chunks: encoder = zfec.Encoder(k, n) outdatalist = encoder.encode(outdatalist) for i in range(len(outdatalist)): if i in failed: filename = dest[i] outfile = open(filename, 'wb') outfile.write(outdatalist[i]) outfile.close()
def set_params(self, data_size, required_shares, max_shares): assert required_shares <= max_shares self.data_size = data_size self.required_shares = required_shares self.max_shares = max_shares self.share_size = mathutil.div_ceil(data_size, required_shares) self.last_share_padding = mathutil.pad_size(self.share_size, required_shares) self.encoder = zfec.Encoder(required_shares, max_shares)
def encode_file(inf, cb, k, m, chunksize=4096): """ Read in the contents of inf, encode, and call cb with the results. First, k "input blocks" will be read from inf, each input block being of size chunksize. Then these k blocks will be encoded into m "result blocks". Then cb will be invoked, passing a list of the m result blocks as its first argument, and the length of the encoded data as its second argument. (The length of the encoded data is always equal to k*chunksize, until the last iteration, when the end of the file has been reached and less than k*chunksize bytes could be read from the file.) This procedure is iterated until the end of the file is reached, in which case the space of the input blocks that is unused is filled with zeroes before encoding. Note that the sequence passed in calls to cb() contains mutable array objects in its first k elements whose contents will be overwritten when the next segment is read from the input file. Therefore the implementation of cb() has to either be finished with those first k arrays before returning, or if it wants to keep the contents of those arrays for subsequent use after it has returned then it must make a copy of them to keep. @param inf the file object from which to read the data @param cb the callback to be invoked with the results @param k the number of shares required to reconstruct the file @param m the total number of shares created @param chunksize how much data to read from inf for each of the k input blocks """ enc = zfec.Encoder(k, m) l = tuple([ array.array('c') for i in range(k) ]) indatasize = k*chunksize # will be reset to shorter upon EOF eof = False ZEROES=array.array('c', ['\x00'])*chunksize while not eof: # This loop body executes once per segment. i = 0 while (i<len(l)): # This loop body executes once per chunk. a = l[i] del a[:] try: a.fromfile(inf, chunksize) i += 1 except EOFError: eof = True indatasize = i*chunksize + len(a) # padding a.fromstring("\x00" * (chunksize-len(a))) i += 1 while (i<len(l)): a = l[i] a[:] = ZEROES i += 1 res = enc.encode(l) cb(res, indatasize)
def test_bad_args_construct_encoder(self): try: zfec.Encoder(-1, -1) except zfec.Error as e: assert "argument is required to be greater than or equal to 1" in str( e), e else: self.fail( "Should have gotten an exception from out-of-range arguments.") try: zfec.Encoder(1, 257) except zfec.Error as e: assert "argument is required to be less than or equal to 256" in str( e), e else: self.fail( "Should have gotten an exception from out-of-range arguments.")
def test_instantiate_encoder_no_args(self): try: e = zfec.Encoder() except TypeError: # Okay, so that's because we're required to pass constructor args. pass else: # Oops, it should have raised an exception. self.fail("Should have raised exception from incorrect arguments to constructor.")
def test_from_agl_py(self): e = zfec.Encoder(3, 5) b0 = '\x01'*8 ; b1 = '\x02'*8 ; b2 = '\x03'*8 # print "_from_py before encoding:" # print "b0: %s, b1: %s, b2: %s" % tuple(base64.b16encode(x) for x in [b0, b1, b2]) b3, b4 = e.encode([b0, b1, b2], (3, 4)) # print "after encoding:" # print "b3: %s, b4: %s" % tuple(base64.b16encode(x) for x in [b3, b4]) d = zfec.Decoder(3, 5) r0, r1, r2 = d.decode((b2, b3, b4), (1, 2, 3))
def _h(k, m, ss): encer = zfec.Encoder(k, m) nums_and_blocks = list(enumerate(encer.encode(ss))) assert isinstance(nums_and_blocks, list), nums_and_blocks assert len(nums_and_blocks) == m, (len(nums_and_blocks), m,) nums_and_blocks = random.sample(nums_and_blocks, k) blocks = [ x[1] for x in nums_and_blocks ] nums = [ x[0] for x in nums_and_blocks ] decer = zfec.Decoder(k, m) decoded = decer.decode(blocks, nums) assert len(decoded) == len(ss), (len(decoded), len(ss),) assert tuple([str(s) for s in decoded]) == tuple([str(s) for s in ss]), (tuple([ab(str(s)) for s in decoded]), tuple([ab(str(s)) for s in ss]),)
def main(): for i in range(2, 8): N = 2**i t = 2**(i - 2) Threshold = ceil((N - t + 1) / 2.0) zfecEncoder = zfec.Encoder(Threshold, N) zfecDecoder = zfec.Decoder(Threshold, N) for j in range(9, 12): Tx = os.urandom((2**j) * 250) start = time.time() fragList = testEncoder(N, t, Tx, Threshold, zfecEncoder) print N, t, 2**j, 'encode', time.time() - start start = time.time() zfecDecoder.decode(fragList[:Threshold], range(Threshold)) print N, t, 2**j, 'decode', time.time() - start
def _encode_obj(self, data): """Encode a single FEC object Args: data : Data to encode (bytes array) Note: Unlike method encode(), this method must take a data object that fits within 256 chunks with overhead so that it fits into a single FEC object. Furthermore, while encode() returns FEC packets, this method returns FEC chunks. Returns: List of FEC-encoded chunks. """ assert (isinstance(data, bytes)) n_chunks = ceil(len(data) / CHUNK_SIZE) n_overhead_chunks = ceil(self.overhead * n_chunks) n_fec_chunks = n_chunks + n_overhead_chunks assert (n_fec_chunks <= MAX_FEC_CHUNKS) logger.debug("Original Chunks: {} / " "Overhead Chunks: {} / " "Total: {}".format(n_chunks, n_overhead_chunks, n_fec_chunks)) # Split the given data array into chunks chunks = [] for i_chunk in range(n_chunks): # Byte range of the next chunk: s_byte = i_chunk * CHUNK_SIZE # starting byte e_byte = (i_chunk + 1) * CHUNK_SIZE # ending byte chunk = data[s_byte:e_byte] # The last chunk may need zero-padding if (i_chunk + 1 == n_chunks and len(chunk) < CHUNK_SIZE): chunk += bytes(CHUNK_SIZE - len(chunk)) assert (len(chunk) == CHUNK_SIZE) chunks.append(chunk) # Generate the corresponding FEC chunks encoder = zfec.Encoder(n_chunks, n_fec_chunks) return encoder.encode(chunks)
def encode(K, N, m): """Erasure encodes string ``m`` into ``N`` blocks, such that any ``K`` can reconstruct. :param int K: K :param int N: number of blocks to encode string ``m`` into. :param int m: string to encode. :return list: Erasure codes resulting from encoding ``m`` into ``N`` blocks using ``zfec`` lib. """ encoder = zfec.Encoder(K, N) assert K <= 256 # TODO: Record this assumption! # pad m to a multiple of K bytes padlen = K - (len(m) % K) m += padlen * chr(K - padlen) step = len(m) / K blocks = [m[i * step:(i + 1) * step] for i in range(K)] stripes = encoder.encode(blocks) return stripes
def ecEncode(src, dataChunks, parityChunks, setting): '''Encode a file into data and parity chunks by erasure coding.''' '''Use zfec library.''' k = len(dataChunks) n = k + len(parityChunks) #Split src file into data chunks: infile = open(src, 'rb') data = infile.read() infile.close() totalchunk = len(dataChunks) filesize = len(data) if filesize == 0: for i in range(totalchunk): outfile = open(dataChunks[i], 'wb') outfile.close() else: chunksize = filesize/totalchunk + 1 setting.chunksize = chunksize chunkid = 0 for i in range(0, filesize+1, chunksize): outfile = open(dataChunks[chunkid], 'wb') startaddr = i endaddr = i + chunksize outfile.write(data[startaddr:endaddr]) if endaddr > filesize: paddingsize = endaddr - (filesize) for j in range(0, paddingsize): outfile.write('\0') outfile.close() chunkid += 1 #Encode to generate parity chunks: encoder = zfec.Encoder(k, n) indatalist = [] for filename in dataChunks: infile = open(filename, 'rb') indatalist.append(infile.read()) infile.close() outdatalist = encoder.encode(indatalist) for i in range(n-k): outfile = open(parityChunks[i],'wb') outfile.write(outdatalist[k+i]) outfile.close()
def encode_file_stringy(inf, cb, k, m, chunksize=4096): """ Read in the contents of inf, encode, and call cb with the results. First, k "input blocks" will be read from inf, each input block being of size chunksize. Then these k blocks will be encoded into m "result blocks". Then cb will be invoked, passing a list of the m result blocks as its first argument, and the length of the encoded data as its second argument. (The length of the encoded data is always equal to k*chunksize, until the last iteration, when the end of the file has been reached and less than k*chunksize bytes could be read from the file.) This procedure is iterated until the end of the file is reached, in which case the part of the input shares that is unused is filled with zeroes before encoding. @param inf the file object from which to read the data @param cb the callback to be invoked with the results @param k the number of shares required to reconstruct the file @param m the total number of shares created @param chunksize how much data to read from inf for each of the k input blocks """ enc = zfec.Encoder(k, m) indatasize = k*chunksize # will be reset to shorter upon EOF while indatasize == k*chunksize: # This loop body executes once per segment. i = 0 l = [] ZEROES = b'\x00'*chunksize while i<k: # This loop body executes once per chunk. i += 1 l.append(inf.read(chunksize)) if len(l[-1]) < chunksize: indatasize = i*chunksize + len(l[-1]) # padding l[-1] = l[-1] + b"\x00" * (chunksize-len(l[-1])) while i<k: l.append(ZEROES) i += 1 res = enc.encode(l) cb(res, indatasize)
def encode(k, n, m): """Erasure encodes string ``m`` into ``n`` blocks, such that any ``k`` can reconstruct. :param int k: k :param int n: number of blocks to encode string ``m`` into. :param bytes m: bytestring to encode. :return list: Erasure codes resulting from encoding ``m`` into ``n`` blocks using ``zfec`` lib. """ try: m = m.encode() except AttributeError: pass encoder = zfec.Encoder(k, n) assert k <= 256 # TODO: Record this assumption! # pad m to a multiple of K bytes padlen = k - (len(m) % k) m += padlen * chr(k - padlen).encode() step = len(m) // k blocks = [m[i * step:(i + 1) * step] for i in range(k)] stripes = encoder.encode(blocks) return stripes
def multiSigBr(pid, N, t, msg, broadcast, receive, outputs, send): # Since all the parties we have are symmetric, so I implement this function for N instances of A-cast as a whole # Here msg is a set of transactions assert (isinstance(outputs, list)) for i in outputs: assert (isinstance(i, Queue)) keys = getECDSAKeys() Threshold = N - 2 * t Threshold2 = N - t zfecEncoder = zfec.Encoder(Threshold, N) zfecDecoder = zfec.Decoder(Threshold, N) def merkleTree(strList, someHash=coolSHA256Hash): # someHash is a mapping from a int to a int treeLength = 2**ceil(math.log(len(strList)) / math.log(2)) mt = [0] * (treeLength * 2) # find a place to put our leaves for i in range(len(strList)): mt[i + treeLength] = someHash( strList[i] ) # TODO: need to change strList[i] from a string to an integer here. for i in range(treeLength - 1, 0, -1): # 1, 2, 3, ..., treeLength - 1 # mt[i] = someHash(''.join([chr(ord(a) ^ ord(b)) for a, b in zip(mt[i*2], mt[i*2+1])])) # XOR is commutative mt[i] = someHash(mt[i * 2] + mt[i * 2 + 1]) # concat is not commutative return mt def getMerkleBranch(index, mt): res = [] t = index + (len(mt) >> 1) while t > 1: res.append(mt[t ^ 1]) # we are picking up the sibling t /= 2 return res def merkleVerify(val, rootHash, branch, someHash, index): # index has information on whether we are facing a left sibling or a right sibling tmp = someHash(val) tindex = index for br in branch: tmp = someHash((tindex & 1) and br + tmp or tmp + br) tindex >>= 1 if tmp != rootHash: print "Verification failed with", someHash( val), rootHash, branch, tmp == rootHash return tmp == rootHash def Listener(): opinions = [defaultdict(lambda: 0) for _ in range(N)] rootHashes = dict() readyCounter = [defaultdict(lambda: 0) for _ in range(N)] signed = [False] * N readySent = [False] * N reconstDone = [False] * N while True: # main loop sender, msgBundle = receive() if msgBundle[0] == 'i' and not signed[sender]: if keys[sender].verify( sha1hash(''.join([ msgBundle[1][0], msgBundle[1][1], ''.join(msgBundle[1][2]) ])), msgBundle[2]): assert isinstance(msgBundle[1], tuple) if not merkleVerify(msgBundle[1][0], msgBundle[1][1], msgBundle[1][2], coolSHA256Hash, pid): continue if sender in rootHashes: if rootHashes[sender] != msgBundle[1][1]: print "Cheating caught, exiting" sys.exit(0) else: rootHashes[sender] = msgBundle[1][1] newBundle = (sender, msgBundle[1][0], msgBundle[1][1], msgBundle[1][2] ) # assert each frag has a length of step broadcast(('e', newBundle, keys[pid].sign( sha1hash(''.join([ str(newBundle[0]), newBundle[1], newBundle[2], ''.join(newBundle[3]) ]))))) signed[sender] = True else: raise ECDSASignatureError() elif msgBundle[0] == 'e': if keys[sender].verify( sha1hash(''.join([ str(msgBundle[1][0]), msgBundle[1][1], msgBundle[1][2], ''.join(msgBundle[1][3]) ])), msgBundle[2]): originBundle = msgBundle[1] if not merkleVerify(originBundle[1], originBundle[2], originBundle[3], coolSHA256Hash, sender): continue if originBundle[0] in rootHashes: if rootHashes[originBundle[0]] != originBundle[2]: print "Cheating caught, exiting" sys.exit(0) else: rootHashes[originBundle[0]] = originBundle[2] opinions[originBundle[0]][sender] = originBundle[ 1] # We are going to move this part to kekeketktktktk if len(opinions[originBundle[0]] ) >= Threshold2 and not readySent[originBundle[0]]: readySent[originBundle[0]] = True broadcast( ('r', originBundle[0], originBundle[2])) # We are broadcasting its hash else: raise ECDSASignatureError() elif msgBundle[0] == 'r': readyCounter[msgBundle[1]][msgBundle[2]] += 1 tmp = readyCounter[msgBundle[1]][msgBundle[2]] if tmp >= t + 1 and not readySent[msgBundle[1]]: readySent[msgBundle[1]] = True broadcast(('r', msgBundle[1], msgBundle[2])) if tmp >= Threshold2 and not outputs[msgBundle[1]].full() and \ not reconstDone[msgBundle[1]] and len(opinions[msgBundle[1]]) >= Threshold: reconstDone[msgBundle[1]] = True if msgBundle[1] in rootHashes: if rootHashes[msgBundle[1]] != msgBundle[2]: print "Cheating caught, exiting" sys.exit(0) else: rootHashes[msgBundle[1]] = msgBundle[2] if opinions[msgBundle[1]].values()[0] == '': reconstruction = [''] else: reconstruction = zfecDecoder.decode( opinions[msgBundle[1]].values()[:Threshold], opinions[msgBundle[1]].keys()[:Threshold] ) # We only take the first [Threshold] fragments rawbuf = ''.join(reconstruction) buf = rawbuf[:-ord(rawbuf[-1])] # Check root hash step = len( buf ) / Threshold + 1 # len(buf) % Threshold == 0 and len(buf) / Threshold or (len(buf) / Threshold + 1) assert step * Threshold - len(buf) < 256 # assumption buf_ = buf.ljust(step * Threshold - 1, '\xFF') + chr(step * Threshold - len(buf)) fragList = [ buf_[i * step:(i + 1) * step] for i in range(Threshold) ] encodedFragList = zfecEncoder.encode(fragList) mt = merkleTree(encodedFragList, coolSHA256Hash) assert rootHashes[msgBundle[1]] == mt[ 1] # full binary tree if outputs[msgBundle[1]].empty(): outputs[msgBundle[1]].put(buf) greenletPacker(Greenlet(Listener), 'multiSigBr.Listener', (pid, N, t, msg, broadcast, receive, outputs)).start() buf = msg # We already assumed the proposals are byte strings step = len( buf ) / Threshold + 1 # len(buf) % Threshold == 0 and len(buf) / Threshold or (len(buf) / Threshold + 1) assert step * Threshold - len(buf) < 256 # assumption buf = buf.ljust(step * Threshold - 1, '\xFF') + chr(step * Threshold - len(buf)) fragList = [buf[i * step:(i + 1) * step] for i in range(Threshold)] encodedFragList = zfecEncoder.encode(fragList) mt = merkleTree(encodedFragList, coolSHA256Hash) rootHash = mt[1] # full binary tree for i in range(N): mb = getMerkleBranch( i, mt) # notice that index starts from 1 and pid starts from 0 newBundle = (encodedFragList[i], rootHash, mb) send(i, ('i', newBundle, keys[pid].sign( sha1hash(''.join( [newBundle[0], newBundle[1], ''.join(newBundle[2])])))))
def test_bad_args_construct_encoder(self): try: zfec.Encoder(-1, -1) except zfec.Error, e: assert "argument is required to be greater than or equal to 1" in str(e), e
zfec.Decoder(3, 2) except zfec.Error, e: assert "first argument is required to be less than or equal to the second argument" in str(e), e else: self.fail("Should have gotten an exception from out-of-range arguments.") def test_bad_args_construct_encoder(self): try: zfec.Encoder(-1, -1) except zfec.Error, e: assert "argument is required to be greater than or equal to 1" in str(e), e else: self.fail("Should have gotten an exception from out-of-range arguments.") try: zfec.Encoder(1, 257) except zfec.Error, e: assert "argument is required to be less than or equal to 256" in str(e), e else: self.fail("Should have gotten an exception from out-of-range arguments.") def test_bad_args_dec(self): decer = zfec.Decoder(2, 4) try: decer.decode(98, []) # first argument is not a sequence except TypeError, e: assert "First argument was not a sequence" in str(e), e else: self.fail("Should have gotten TypeError for wrong type of second argument.")
def __init__(self, k, m): self.fec = zfec.Encoder(k, m)
import zfec print(zfec.Encoder(1, 9).encode('hi'))
import os import struct from JumpScale import j j.application.start("zfectest") #@todo NOT WORKING, need to implement import zfec encoder = zfec.Encoder(10, 14) filename = "/var/lib/libvirt/images/Ubuntu.img" counter = 0 def encode(counter, series): counter += 1 counterSeries = 0 seriesOut = encoder.encode(series) for out in seriesOut: counterSeries += 1 pathout = "out/Ubuntu.img.%s_%s" % (counter, counterSeries) j.system.fs.writeFile(pathout, out) print "%s_%s" % (counter, counterSeries) return counter size = 0 series = []
def __init__(self, k, m): self.k = k self.m = m self.z_encoder = zfec.Encoder(k=k, m=m) self.z_decoder = zfec.Decoder(k=k, m=m)