def test_bad_args_construct_decoder(self): try: zfec.Decoder(-1, -1) except zfec.Error as e: assert "argument is required to be greater than or equal to 1" in str( e), e else: self.fail( "Should have gotten an exception from out-of-range arguments.") try: zfec.Decoder(1, 257) except zfec.Error as e: assert "argument is required to be less than or equal to 256" in str( e), e else: self.fail( "Should have gotten an exception from out-of-range arguments.") try: zfec.Decoder(3, 2) except zfec.Error as e: assert "first argument is required to be less than or equal to the second argument" in str( e), e else: self.fail( "Should have gotten an exception from out-of-range arguments.")
def test_bad_args_dec(self): decer = zfec.Decoder(2, 4) try: decer.decode(98, []) # first argument is not a sequence except TypeError, e: assert "First argument was not a sequence" in str(e), e
def ecDecodeChunks(n, k, src, blocknums, dest, setting, failed=[], nativeFailed=True): '''Generate a full list of chunks.''' '''Use zfec library.''' if nativeFailed: decoder = zfec.Decoder(k, n) indatalist = [] for filename in src: infile = open(filename, 'rb') indatalist.append(infile.read()) infile.close() #Get native data chunks: outdatalist = decoder.decode(indatalist, blocknums) else: outdatalist = [] for filename in src: infile = open(filename, 'rb') outdatalist.append(infile.read()) infile.close() #Generate full list of chunks: encoder = zfec.Encoder(k, n) outdatalist = encoder.encode(outdatalist) for i in range(len(outdatalist)): if i in failed: filename = dest[i] outfile = open(filename, 'wb') outfile.write(outdatalist[i]) outfile.close()
def decode(k, n, stripes): """Decodes an erasure-encoded string from a subset of stripes :param list stripes: a container of :math:`n` elements, each of which is either a string or ``None`` at least :math:`k` elements are strings all string elements are the same length """ assert len(stripes) == n blocks = [] blocknums = [] for i, block in enumerate(stripes): if block is None: continue blocks.append(block) blocknums.append(i) if len(blocks) == k: break else: raise ValueError("Too few to recover") decoder = zfec.Decoder(k, n) rec = decoder.decode(blocks, blocknums) m = b"".join(rec) padlen = k - m[-1] m = m[:-padlen] return m
def decode(self): if self.__frag_recv == self.frag_count: # No error FEC decoding necessary return b''.join([self.__fragments[s] for s in range(self.frag_count)]) k = self.frag_count n = k + self.fec_count decoder = zfec.Decoder(k, n) sharenums = list(self.__fragments.keys()) return b''.join(decoder.decode([self.__fragments[s] for s in sharenums], sharenums))
def set_params(self, data_size, required_shares, max_shares): self.data_size = data_size self.required_shares = required_shares self.max_shares = max_shares self.chunk_size = self.required_shares self.num_chunks = mathutil.div_ceil(self.data_size, self.chunk_size) self.share_size = self.num_chunks self.decoder = zfec.Decoder(self.required_shares, self.max_shares)
def test_instantiate_decoder_no_args(self): try: e = zfec.Decoder() except TypeError: # Okay, so that's because we're required to pass constructor args. pass else: # Oops, it should have raised an exception. self.fail("Should have raised exception from incorrect arguments to constructor.")
def _h(k, m, ss): encer = zfec.Encoder(k, m) nums_and_blocks = list(enumerate(encer.encode(ss))) assert isinstance(nums_and_blocks, list), nums_and_blocks assert len(nums_and_blocks) == m, (len(nums_and_blocks), m,) nums_and_blocks = random.sample(nums_and_blocks, k) blocks = [ x[1] for x in nums_and_blocks ] nums = [ x[0] for x in nums_and_blocks ] decer = zfec.Decoder(k, m) decoded = decer.decode(blocks, nums) assert len(decoded) == len(ss), (len(decoded), len(ss),) assert tuple([str(s) for s in decoded]) == tuple([str(s) for s in ss]), (tuple([ab(str(s)) for s in decoded]), tuple([ab(str(s)) for s in ss]),)
def test_from_agl_py(self): e = zfec.Encoder(3, 5) b0 = '\x01'*8 ; b1 = '\x02'*8 ; b2 = '\x03'*8 # print "_from_py before encoding:" # print "b0: %s, b1: %s, b2: %s" % tuple(base64.b16encode(x) for x in [b0, b1, b2]) b3, b4 = e.encode([b0, b1, b2], (3, 4)) # print "after encoding:" # print "b3: %s, b4: %s" % tuple(base64.b16encode(x) for x in [b3, b4]) d = zfec.Decoder(3, 5) r0, r1, r2 = d.decode((b2, b3, b4), (1, 2, 3))
def main(): for i in range(2, 8): N = 2**i t = 2**(i - 2) Threshold = ceil((N - t + 1) / 2.0) zfecEncoder = zfec.Encoder(Threshold, N) zfecDecoder = zfec.Decoder(Threshold, N) for j in range(9, 12): Tx = os.urandom((2**j) * 250) start = time.time() fragList = testEncoder(N, t, Tx, Threshold, zfecEncoder) print N, t, 2**j, 'encode', time.time() - start start = time.time() zfecDecoder.decode(fragList[:Threshold], range(Threshold)) print N, t, 2**j, 'decode', time.time() - start
def decode(self): if self.__frag_recv == self.frag_count: # No error FEC decoding necessary return b''.join( [self.__fragments[s] for s in range(self.frag_count)]) k = self.frag_count n = k + self.fec_count decoder = zfec.Decoder(k, n) sharenums = list(self.__fragments.keys()) if len(sharenums) != k: print( "[ERROR] Unexpected number of fragments. k = {}, sharenums = {}" .format(k, sharenums)) return return b''.join( decoder.decode([self.__fragments[s] for s in sharenums], sharenums))
def _decode_obj(self, obj_len, chunks, chunk_ids): """Decode a single FEC object Args: obj_len : Length of the original object encoded by the FEC object. chunks : List of FEC chunks. chunk_ids : List of ids associated with the list of FEC chunks. Note: Unlike method decode(), this method processes a single encoded FEC object. Returns: (bytearray) with the decoded (original) data object. """ # The original (uncoded) object must fit within the maximum number of # FEC chunks. Most of the time `obj_len` will be less than the # maximum. It only hits the maximum if the FEC overhead is set to zero. assert (obj_len <= (MAX_FEC_CHUNKS * CHUNK_SIZE)) # Number of FEC chunks required to decode the original message: n_chunks = ceil(obj_len / CHUNK_SIZE) # FEC decoder decoder = zfec.Decoder(n_chunks, MAX_FEC_CHUNKS) # NOTE: The hard-coded "MAX_FEC_CHUNKS" represents the maximum number # of chunks that can be generated. The receiver does not know how many # chunks the sender really generated. Nevertheless, it does not need to # know, as long as it receives at least n_chunks (any combination of # n_chunks). The explanation for this requirement is that the FEC # scheme is a maximum distance separable (MDS) erasure code. # Decode using the minimum required number of FEC chunks decoded_chunks = decoder.decode(chunks[:n_chunks], chunk_ids[:n_chunks]) # Concatenate the decoded chunks to form the original object decoded_obj = bytearray() for chunk in decoded_chunks: assert (len(chunk) == CHUNK_SIZE) decoded_obj += chunk # Remove any zero-padding that may have been applied to the last chunk: return decoded_obj[:obj_len]
def ecDecodeFile(n, k, src, blocknums, dest, filesize, setting): '''Decode a list of chunks to original file by erasure coding.''' '''Use zfec library.''' decoder = zfec.Decoder(k, n) indatalist = [] for filename in src: infile = open(filename, 'rb') indatalist.append(infile.read()) infile.close() #Get native data chunks: outdatalist = decoder.decode(indatalist, blocknums) outfile = open(dest,'wb') writesize = 0 #Join chunks into file: for data in outdatalist: writesize += len(data) if writesize > filesize: chunksize = len(data) - (writesize - filesize) outfile.write(data[0:chunksize]) else: outfile.write(data) outfile.close()
def test_bad_args_dec(self): decer = zfec.Decoder(2, 4) try: decer.decode(98, []) # first argument is not a sequence except TypeError as e: assert "First argument was not a sequence" in str(e), e else: self.fail( "Should have gotten TypeError for wrong type of second argument." ) try: decer.decode([ "a", "b", ], [ "c", "d", ]) except zfec.Error as e: assert "Precondition violation: second argument is required to contain int" in str( e), e else: self.fail( "Should have gotten zfec.Error for wrong type of second argument." ) try: decer.decode([ "a", "b", ], 98) # not a sequence at all except TypeError as e: assert "Second argument was not a sequence" in str(e), e else: self.fail( "Should have gotten TypeError for wrong type of second argument." )
def __init__(self, k, m): self.k = k self.m = m self.z_encoder = zfec.Encoder(k=k, m=m) self.z_decoder = zfec.Decoder(k=k, m=m)
def __init__(self, k, m): self.fec = zfec.Decoder(k, m)
def test_bad_args_construct_decoder(self): try: zfec.Decoder(-1, -1) except zfec.Error, e: assert "argument is required to be greater than or equal to 1" in str(e), e
def test_random(self): for i in range(3): _help_test_random() if VERBOSE: print "%d randomized tests pass." % (i+1) def test_bad_args_construct_decoder(self): try: zfec.Decoder(-1, -1) except zfec.Error, e: assert "argument is required to be greater than or equal to 1" in str(e), e else: self.fail("Should have gotten an exception from out-of-range arguments.") try: zfec.Decoder(1, 257) except zfec.Error, e: assert "argument is required to be less than or equal to 256" in str(e), e else: self.fail("Should have gotten an exception from out-of-range arguments.") try: zfec.Decoder(3, 2) except zfec.Error, e: assert "first argument is required to be less than or equal to the second argument" in str(e), e else: self.fail("Should have gotten an exception from out-of-range arguments.") def test_bad_args_construct_encoder(self): try: zfec.Encoder(-1, -1)
def multiSigBr(pid, N, t, msg, broadcast, receive, outputs, send): # Since all the parties we have are symmetric, so I implement this function for N instances of A-cast as a whole # Here msg is a set of transactions assert (isinstance(outputs, list)) for i in outputs: assert (isinstance(i, Queue)) keys = getECDSAKeys() Threshold = N - 2 * t Threshold2 = N - t zfecEncoder = zfec.Encoder(Threshold, N) zfecDecoder = zfec.Decoder(Threshold, N) def merkleTree(strList, someHash=coolSHA256Hash): # someHash is a mapping from a int to a int treeLength = 2**ceil(math.log(len(strList)) / math.log(2)) mt = [0] * (treeLength * 2) # find a place to put our leaves for i in range(len(strList)): mt[i + treeLength] = someHash( strList[i] ) # TODO: need to change strList[i] from a string to an integer here. for i in range(treeLength - 1, 0, -1): # 1, 2, 3, ..., treeLength - 1 # mt[i] = someHash(''.join([chr(ord(a) ^ ord(b)) for a, b in zip(mt[i*2], mt[i*2+1])])) # XOR is commutative mt[i] = someHash(mt[i * 2] + mt[i * 2 + 1]) # concat is not commutative return mt def getMerkleBranch(index, mt): res = [] t = index + (len(mt) >> 1) while t > 1: res.append(mt[t ^ 1]) # we are picking up the sibling t /= 2 return res def merkleVerify(val, rootHash, branch, someHash, index): # index has information on whether we are facing a left sibling or a right sibling tmp = someHash(val) tindex = index for br in branch: tmp = someHash((tindex & 1) and br + tmp or tmp + br) tindex >>= 1 if tmp != rootHash: print "Verification failed with", someHash( val), rootHash, branch, tmp == rootHash return tmp == rootHash def Listener(): opinions = [defaultdict(lambda: 0) for _ in range(N)] rootHashes = dict() readyCounter = [defaultdict(lambda: 0) for _ in range(N)] signed = [False] * N readySent = [False] * N reconstDone = [False] * N while True: # main loop sender, msgBundle = receive() if msgBundle[0] == 'i' and not signed[sender]: if keys[sender].verify( sha1hash(''.join([ msgBundle[1][0], msgBundle[1][1], ''.join(msgBundle[1][2]) ])), msgBundle[2]): assert isinstance(msgBundle[1], tuple) if not merkleVerify(msgBundle[1][0], msgBundle[1][1], msgBundle[1][2], coolSHA256Hash, pid): continue if sender in rootHashes: if rootHashes[sender] != msgBundle[1][1]: print "Cheating caught, exiting" sys.exit(0) else: rootHashes[sender] = msgBundle[1][1] newBundle = (sender, msgBundle[1][0], msgBundle[1][1], msgBundle[1][2] ) # assert each frag has a length of step broadcast(('e', newBundle, keys[pid].sign( sha1hash(''.join([ str(newBundle[0]), newBundle[1], newBundle[2], ''.join(newBundle[3]) ]))))) signed[sender] = True else: raise ECDSASignatureError() elif msgBundle[0] == 'e': if keys[sender].verify( sha1hash(''.join([ str(msgBundle[1][0]), msgBundle[1][1], msgBundle[1][2], ''.join(msgBundle[1][3]) ])), msgBundle[2]): originBundle = msgBundle[1] if not merkleVerify(originBundle[1], originBundle[2], originBundle[3], coolSHA256Hash, sender): continue if originBundle[0] in rootHashes: if rootHashes[originBundle[0]] != originBundle[2]: print "Cheating caught, exiting" sys.exit(0) else: rootHashes[originBundle[0]] = originBundle[2] opinions[originBundle[0]][sender] = originBundle[ 1] # We are going to move this part to kekeketktktktk if len(opinions[originBundle[0]] ) >= Threshold2 and not readySent[originBundle[0]]: readySent[originBundle[0]] = True broadcast( ('r', originBundle[0], originBundle[2])) # We are broadcasting its hash else: raise ECDSASignatureError() elif msgBundle[0] == 'r': readyCounter[msgBundle[1]][msgBundle[2]] += 1 tmp = readyCounter[msgBundle[1]][msgBundle[2]] if tmp >= t + 1 and not readySent[msgBundle[1]]: readySent[msgBundle[1]] = True broadcast(('r', msgBundle[1], msgBundle[2])) if tmp >= Threshold2 and not outputs[msgBundle[1]].full() and \ not reconstDone[msgBundle[1]] and len(opinions[msgBundle[1]]) >= Threshold: reconstDone[msgBundle[1]] = True if msgBundle[1] in rootHashes: if rootHashes[msgBundle[1]] != msgBundle[2]: print "Cheating caught, exiting" sys.exit(0) else: rootHashes[msgBundle[1]] = msgBundle[2] if opinions[msgBundle[1]].values()[0] == '': reconstruction = [''] else: reconstruction = zfecDecoder.decode( opinions[msgBundle[1]].values()[:Threshold], opinions[msgBundle[1]].keys()[:Threshold] ) # We only take the first [Threshold] fragments rawbuf = ''.join(reconstruction) buf = rawbuf[:-ord(rawbuf[-1])] # Check root hash step = len( buf ) / Threshold + 1 # len(buf) % Threshold == 0 and len(buf) / Threshold or (len(buf) / Threshold + 1) assert step * Threshold - len(buf) < 256 # assumption buf_ = buf.ljust(step * Threshold - 1, '\xFF') + chr(step * Threshold - len(buf)) fragList = [ buf_[i * step:(i + 1) * step] for i in range(Threshold) ] encodedFragList = zfecEncoder.encode(fragList) mt = merkleTree(encodedFragList, coolSHA256Hash) assert rootHashes[msgBundle[1]] == mt[ 1] # full binary tree if outputs[msgBundle[1]].empty(): outputs[msgBundle[1]].put(buf) greenletPacker(Greenlet(Listener), 'multiSigBr.Listener', (pid, N, t, msg, broadcast, receive, outputs)).start() buf = msg # We already assumed the proposals are byte strings step = len( buf ) / Threshold + 1 # len(buf) % Threshold == 0 and len(buf) / Threshold or (len(buf) / Threshold + 1) assert step * Threshold - len(buf) < 256 # assumption buf = buf.ljust(step * Threshold - 1, '\xFF') + chr(step * Threshold - len(buf)) fragList = [buf[i * step:(i + 1) * step] for i in range(Threshold)] encodedFragList = zfecEncoder.encode(fragList) mt = merkleTree(encodedFragList, coolSHA256Hash) rootHash = mt[1] # full binary tree for i in range(N): mb = getMerkleBranch( i, mt) # notice that index starts from 1 and pid starts from 0 newBundle = (encodedFragList[i], rootHash, mb) send(i, ('i', newBundle, keys[pid].sign( sha1hash(''.join( [newBundle[0], newBundle[1], ''.join(newBundle[2])])))))