def test_bad_args_construct_decoder(self):
try:
zfec.Decoder(-1, -1)
except zfec.Error as e:
assert "argument is required to be greater than or equal to 1" in str(
e), e
else:
self.fail(
"Should have gotten an exception from out-of-range arguments.")
try:
zfec.Decoder(1, 257)
except zfec.Error as e:
assert "argument is required to be less than or equal to 256" in str(
e), e
else:
self.fail(
"Should have gotten an exception from out-of-range arguments.")
try:
zfec.Decoder(3, 2)
except zfec.Error as e:
assert "first argument is required to be less than or equal to the second argument" in str(
e), e
else:
self.fail(
"Should have gotten an exception from out-of-range arguments.")
def test_bad_args_dec(self):
decer = zfec.Decoder(2, 4)
try:
decer.decode(98, []) # first argument is not a sequence
except TypeError, e:
assert "First argument was not a sequence" in str(e), e
def ecDecodeChunks(n, k, src, blocknums, dest, setting, failed=[], nativeFailed=True):
'''Generate a full list of chunks.'''
'''Use zfec library.'''
if nativeFailed:
decoder = zfec.Decoder(k, n)
indatalist = []
for filename in src:
infile = open(filename, 'rb')
indatalist.append(infile.read())
infile.close()
#Get native data chunks:
outdatalist = decoder.decode(indatalist, blocknums)
else:
outdatalist = []
for filename in src:
infile = open(filename, 'rb')
outdatalist.append(infile.read())
infile.close()
#Generate full list of chunks:
encoder = zfec.Encoder(k, n)
outdatalist = encoder.encode(outdatalist)
for i in range(len(outdatalist)):
if i in failed:
filename = dest[i]
outfile = open(filename, 'wb')
outfile.write(outdatalist[i])
outfile.close()
def decode(k, n, stripes):
"""Decodes an erasure-encoded string from a subset of stripes
:param list stripes: a container of :math:`n` elements,
each of which is either a string or ``None``
at least :math:`k` elements are strings
all string elements are the same length
"""
assert len(stripes) == n
blocks = []
blocknums = []
for i, block in enumerate(stripes):
if block is None:
continue
blocks.append(block)
blocknums.append(i)
if len(blocks) == k:
break
else:
raise ValueError("Too few to recover")
decoder = zfec.Decoder(k, n)
rec = decoder.decode(blocks, blocknums)
m = b"".join(rec)
padlen = k - m[-1]
m = m[:-padlen]
return m
def decode(self):
if self.__frag_recv == self.frag_count: # No error FEC decoding necessary
return b''.join([self.__fragments[s] for s in range(self.frag_count)])
k = self.frag_count
n = k + self.fec_count
decoder = zfec.Decoder(k, n)
sharenums = list(self.__fragments.keys())
return b''.join(decoder.decode([self.__fragments[s] for s in sharenums], sharenums))
def set_params(self, data_size, required_shares, max_shares):
self.data_size = data_size
self.required_shares = required_shares
self.max_shares = max_shares
self.chunk_size = self.required_shares
self.num_chunks = mathutil.div_ceil(self.data_size, self.chunk_size)
self.share_size = self.num_chunks
self.decoder = zfec.Decoder(self.required_shares, self.max_shares)
def test_instantiate_decoder_no_args(self):
try:
e = zfec.Decoder()
except TypeError:
# Okay, so that's because we're required to pass constructor args.
pass
else:
# Oops, it should have raised an exception.
self.fail("Should have raised exception from incorrect arguments to constructor.")
def _h(k, m, ss):
encer = zfec.Encoder(k, m)
nums_and_blocks = list(enumerate(encer.encode(ss)))
assert isinstance(nums_and_blocks, list), nums_and_blocks
assert len(nums_and_blocks) == m, (len(nums_and_blocks), m,)
nums_and_blocks = random.sample(nums_and_blocks, k)
blocks = [ x[1] for x in nums_and_blocks ]
nums = [ x[0] for x in nums_and_blocks ]
decer = zfec.Decoder(k, m)
decoded = decer.decode(blocks, nums)
assert len(decoded) == len(ss), (len(decoded), len(ss),)
assert tuple([str(s) for s in decoded]) == tuple([str(s) for s in ss]), (tuple([ab(str(s)) for s in decoded]), tuple([ab(str(s)) for s in ss]),)
def test_from_agl_py(self):
e = zfec.Encoder(3, 5)
b0 = '\x01'*8 ; b1 = '\x02'*8 ; b2 = '\x03'*8
# print "_from_py before encoding:"
# print "b0: %s, b1: %s, b2: %s" % tuple(base64.b16encode(x) for x in [b0, b1, b2])
b3, b4 = e.encode([b0, b1, b2], (3, 4))
# print "after encoding:"
# print "b3: %s, b4: %s" % tuple(base64.b16encode(x) for x in [b3, b4])
d = zfec.Decoder(3, 5)
r0, r1, r2 = d.decode((b2, b3, b4), (1, 2, 3))
def main():
for i in range(2, 8):
N = 2**i
t = 2**(i - 2)
Threshold = ceil((N - t + 1) / 2.0)
zfecEncoder = zfec.Encoder(Threshold, N)
zfecDecoder = zfec.Decoder(Threshold, N)
for j in range(9, 12):
Tx = os.urandom((2**j) * 250)
start = time.time()
fragList = testEncoder(N, t, Tx, Threshold, zfecEncoder)
print N, t, 2**j, 'encode', time.time() - start
start = time.time()
zfecDecoder.decode(fragList[:Threshold], range(Threshold))
print N, t, 2**j, 'decode', time.time() - start
def decode(self):
if self.__frag_recv == self.frag_count: # No error FEC decoding necessary
return b''.join(
[self.__fragments[s] for s in range(self.frag_count)])
k = self.frag_count
n = k + self.fec_count
decoder = zfec.Decoder(k, n)
sharenums = list(self.__fragments.keys())
if len(sharenums) != k:
print(
"[ERROR] Unexpected number of fragments. k = {}, sharenums = {}"
.format(k, sharenums))
return
return b''.join(
decoder.decode([self.__fragments[s] for s in sharenums],
sharenums))
def _decode_obj(self, obj_len, chunks, chunk_ids):
"""Decode a single FEC object
Args:
obj_len : Length of the original object encoded by the FEC
object.
chunks : List of FEC chunks.
chunk_ids : List of ids associated with the list of FEC chunks.
Note:
Unlike method decode(), this method processes a single encoded FEC
object.
Returns:
(bytearray) with the decoded (original) data object.
"""
# The original (uncoded) object must fit within the maximum number of
# FEC chunks. Most of the time `obj_len` will be less than the
# maximum. It only hits the maximum if the FEC overhead is set to zero.
assert (obj_len <= (MAX_FEC_CHUNKS * CHUNK_SIZE))
# Number of FEC chunks required to decode the original message:
n_chunks = ceil(obj_len / CHUNK_SIZE)
# FEC decoder
decoder = zfec.Decoder(n_chunks, MAX_FEC_CHUNKS)
# NOTE: The hard-coded "MAX_FEC_CHUNKS" represents the maximum number
# of chunks that can be generated. The receiver does not know how many
# chunks the sender really generated. Nevertheless, it does not need to
# know, as long as it receives at least n_chunks (any combination of
# n_chunks). The explanation for this requirement is that the FEC
# scheme is a maximum distance separable (MDS) erasure code.
# Decode using the minimum required number of FEC chunks
decoded_chunks = decoder.decode(chunks[:n_chunks],
chunk_ids[:n_chunks])
# Concatenate the decoded chunks to form the original object
decoded_obj = bytearray()
for chunk in decoded_chunks:
assert (len(chunk) == CHUNK_SIZE)
decoded_obj += chunk
# Remove any zero-padding that may have been applied to the last chunk:
return decoded_obj[:obj_len]
def ecDecodeFile(n, k, src, blocknums, dest, filesize, setting):
'''Decode a list of chunks to original file by erasure coding.'''
'''Use zfec library.'''
decoder = zfec.Decoder(k, n)
indatalist = []
for filename in src:
infile = open(filename, 'rb')
indatalist.append(infile.read())
infile.close()
#Get native data chunks:
outdatalist = decoder.decode(indatalist, blocknums)
outfile = open(dest,'wb')
writesize = 0
#Join chunks into file:
for data in outdatalist:
writesize += len(data)
if writesize > filesize:
chunksize = len(data) - (writesize - filesize)
outfile.write(data[0:chunksize])
else:
outfile.write(data)
outfile.close()
def test_bad_args_dec(self):
decer = zfec.Decoder(2, 4)
try:
decer.decode(98, []) # first argument is not a sequence
except TypeError as e:
assert "First argument was not a sequence" in str(e), e
else:
self.fail(
"Should have gotten TypeError for wrong type of second argument."
)
try:
decer.decode([
"a",
"b",
], [
"c",
"d",
])
except zfec.Error as e:
assert "Precondition violation: second argument is required to contain int" in str(
e), e
else:
self.fail(
"Should have gotten zfec.Error for wrong type of second argument."
)
try:
decer.decode([
"a",
"b",
], 98) # not a sequence at all
except TypeError as e:
assert "Second argument was not a sequence" in str(e), e
else:
self.fail(
"Should have gotten TypeError for wrong type of second argument."
)
def __init__(self, k, m):
self.k = k
self.m = m
self.z_encoder = zfec.Encoder(k=k, m=m)
self.z_decoder = zfec.Decoder(k=k, m=m)
def __init__(self, k, m):
self.fec = zfec.Decoder(k, m)
def test_bad_args_construct_decoder(self):
try:
zfec.Decoder(-1, -1)
except zfec.Error, e:
assert "argument is required to be greater than or equal to 1" in str(e), e
def test_random(self):
for i in range(3):
_help_test_random()
if VERBOSE:
print "%d randomized tests pass." % (i+1)
def test_bad_args_construct_decoder(self):
try:
zfec.Decoder(-1, -1)
except zfec.Error, e:
assert "argument is required to be greater than or equal to 1" in str(e), e
else:
self.fail("Should have gotten an exception from out-of-range arguments.")
try:
zfec.Decoder(1, 257)
except zfec.Error, e:
assert "argument is required to be less than or equal to 256" in str(e), e
else:
self.fail("Should have gotten an exception from out-of-range arguments.")
try:
zfec.Decoder(3, 2)
except zfec.Error, e:
assert "first argument is required to be less than or equal to the second argument" in str(e), e
else:
self.fail("Should have gotten an exception from out-of-range arguments.")
def test_bad_args_construct_encoder(self):
try:
zfec.Encoder(-1, -1)
def multiSigBr(pid, N, t, msg, broadcast, receive, outputs, send):
# Since all the parties we have are symmetric, so I implement this function for N instances of A-cast as a whole
# Here msg is a set of transactions
assert (isinstance(outputs, list))
for i in outputs:
assert (isinstance(i, Queue))
keys = getECDSAKeys()
Threshold = N - 2 * t
Threshold2 = N - t
zfecEncoder = zfec.Encoder(Threshold, N)
zfecDecoder = zfec.Decoder(Threshold, N)
def merkleTree(strList, someHash=coolSHA256Hash):
# someHash is a mapping from a int to a int
treeLength = 2**ceil(math.log(len(strList)) / math.log(2))
mt = [0] * (treeLength * 2) # find a place to put our leaves
for i in range(len(strList)):
mt[i + treeLength] = someHash(
strList[i]
) # TODO: need to change strList[i] from a string to an integer here.
for i in range(treeLength - 1, 0, -1): # 1, 2, 3, ..., treeLength - 1
# mt[i] = someHash(''.join([chr(ord(a) ^ ord(b)) for a, b in zip(mt[i*2], mt[i*2+1])])) # XOR is commutative
mt[i] = someHash(mt[i * 2] +
mt[i * 2 + 1]) # concat is not commutative
return mt
def getMerkleBranch(index, mt):
res = []
t = index + (len(mt) >> 1)
while t > 1:
res.append(mt[t ^ 1]) # we are picking up the sibling
t /= 2
return res
def merkleVerify(val, rootHash, branch, someHash, index):
# index has information on whether we are facing a left sibling or a right sibling
tmp = someHash(val)
tindex = index
for br in branch:
tmp = someHash((tindex & 1) and br + tmp or tmp + br)
tindex >>= 1
if tmp != rootHash:
print "Verification failed with", someHash(
val), rootHash, branch, tmp == rootHash
return tmp == rootHash
def Listener():
opinions = [defaultdict(lambda: 0) for _ in range(N)]
rootHashes = dict()
readyCounter = [defaultdict(lambda: 0) for _ in range(N)]
signed = [False] * N
readySent = [False] * N
reconstDone = [False] * N
while True: # main loop
sender, msgBundle = receive()
if msgBundle[0] == 'i' and not signed[sender]:
if keys[sender].verify(
sha1hash(''.join([
msgBundle[1][0], msgBundle[1][1],
''.join(msgBundle[1][2])
])), msgBundle[2]):
assert isinstance(msgBundle[1], tuple)
if not merkleVerify(msgBundle[1][0], msgBundle[1][1],
msgBundle[1][2], coolSHA256Hash, pid):
continue
if sender in rootHashes:
if rootHashes[sender] != msgBundle[1][1]:
print "Cheating caught, exiting"
sys.exit(0)
else:
rootHashes[sender] = msgBundle[1][1]
newBundle = (sender, msgBundle[1][0], msgBundle[1][1],
msgBundle[1][2]
) # assert each frag has a length of step
broadcast(('e', newBundle, keys[pid].sign(
sha1hash(''.join([
str(newBundle[0]), newBundle[1], newBundle[2],
''.join(newBundle[3])
])))))
signed[sender] = True
else:
raise ECDSASignatureError()
elif msgBundle[0] == 'e':
if keys[sender].verify(
sha1hash(''.join([
str(msgBundle[1][0]), msgBundle[1][1],
msgBundle[1][2], ''.join(msgBundle[1][3])
])), msgBundle[2]):
originBundle = msgBundle[1]
if not merkleVerify(originBundle[1], originBundle[2],
originBundle[3], coolSHA256Hash,
sender):
continue
if originBundle[0] in rootHashes:
if rootHashes[originBundle[0]] != originBundle[2]:
print "Cheating caught, exiting"
sys.exit(0)
else:
rootHashes[originBundle[0]] = originBundle[2]
opinions[originBundle[0]][sender] = originBundle[
1] # We are going to move this part to kekeketktktktk
if len(opinions[originBundle[0]]
) >= Threshold2 and not readySent[originBundle[0]]:
readySent[originBundle[0]] = True
broadcast(
('r', originBundle[0],
originBundle[2])) # We are broadcasting its hash
else:
raise ECDSASignatureError()
elif msgBundle[0] == 'r':
readyCounter[msgBundle[1]][msgBundle[2]] += 1
tmp = readyCounter[msgBundle[1]][msgBundle[2]]
if tmp >= t + 1 and not readySent[msgBundle[1]]:
readySent[msgBundle[1]] = True
broadcast(('r', msgBundle[1], msgBundle[2]))
if tmp >= Threshold2 and not outputs[msgBundle[1]].full() and \
not reconstDone[msgBundle[1]] and len(opinions[msgBundle[1]]) >= Threshold:
reconstDone[msgBundle[1]] = True
if msgBundle[1] in rootHashes:
if rootHashes[msgBundle[1]] != msgBundle[2]:
print "Cheating caught, exiting"
sys.exit(0)
else:
rootHashes[msgBundle[1]] = msgBundle[2]
if opinions[msgBundle[1]].values()[0] == '':
reconstruction = ['']
else:
reconstruction = zfecDecoder.decode(
opinions[msgBundle[1]].values()[:Threshold],
opinions[msgBundle[1]].keys()[:Threshold]
) # We only take the first [Threshold] fragments
rawbuf = ''.join(reconstruction)
buf = rawbuf[:-ord(rawbuf[-1])]
# Check root hash
step = len(
buf
) / Threshold + 1 # len(buf) % Threshold == 0 and len(buf) / Threshold or (len(buf) / Threshold + 1)
assert step * Threshold - len(buf) < 256 # assumption
buf_ = buf.ljust(step * Threshold - 1,
'\xFF') + chr(step * Threshold - len(buf))
fragList = [
buf_[i * step:(i + 1) * step] for i in range(Threshold)
]
encodedFragList = zfecEncoder.encode(fragList)
mt = merkleTree(encodedFragList, coolSHA256Hash)
assert rootHashes[msgBundle[1]] == mt[
1] # full binary tree
if outputs[msgBundle[1]].empty():
outputs[msgBundle[1]].put(buf)
greenletPacker(Greenlet(Listener), 'multiSigBr.Listener',
(pid, N, t, msg, broadcast, receive, outputs)).start()
buf = msg # We already assumed the proposals are byte strings
step = len(
buf
) / Threshold + 1 # len(buf) % Threshold == 0 and len(buf) / Threshold or (len(buf) / Threshold + 1)
assert step * Threshold - len(buf) < 256 # assumption
buf = buf.ljust(step * Threshold - 1,
'\xFF') + chr(step * Threshold - len(buf))
fragList = [buf[i * step:(i + 1) * step] for i in range(Threshold)]
encodedFragList = zfecEncoder.encode(fragList)
mt = merkleTree(encodedFragList, coolSHA256Hash)
rootHash = mt[1] # full binary tree
for i in range(N):
mb = getMerkleBranch(
i, mt) # notice that index starts from 1 and pid starts from 0
newBundle = (encodedFragList[i], rootHash, mb)
send(i, ('i', newBundle, keys[pid].sign(
sha1hash(''.join(
[newBundle[0], newBundle[1], ''.join(newBundle[2])])))))
