def encode_file_not_really_and_hash(inf, cb, k, m, chunksize=4096):
hasher = sha1.new()
enc = zfec.Encoder(k, m)
l = tuple([ array.array('c') for i in range(k) ])
indatasize = k*chunksize # will be reset to shorter upon EOF
eof = False
ZEROES=array.array('c', ['\x00'])*chunksize
while not eof:
# This loop body executes once per segment.
i = 0
while (i<len(l)):
# This loop body executes once per chunk.
a = l[i]
del a[:]
try:
a.fromfile(inf, chunksize)
i += 1
except EOFError:
eof = True
indatasize = i*chunksize + len(a)
# padding
a.fromstring("\x00" * (chunksize-len(a)))
i += 1
while (i<len(l)):
a = l[i]
a[:] = ZEROES
i += 1
# res = enc.encode(l)
for thing in l:
hasher.update(thing)
cb(None, None)
def ecDecodeChunks(n, k, src, blocknums, dest, setting, failed=[], nativeFailed=True):
'''Generate a full list of chunks.'''
'''Use zfec library.'''
if nativeFailed:
decoder = zfec.Decoder(k, n)
indatalist = []
for filename in src:
infile = open(filename, 'rb')
indatalist.append(infile.read())
infile.close()
#Get native data chunks:
outdatalist = decoder.decode(indatalist, blocknums)
else:
outdatalist = []
for filename in src:
infile = open(filename, 'rb')
outdatalist.append(infile.read())
infile.close()
#Generate full list of chunks:
encoder = zfec.Encoder(k, n)
outdatalist = encoder.encode(outdatalist)
for i in range(len(outdatalist)):
if i in failed:
filename = dest[i]
outfile = open(filename, 'wb')
outfile.write(outdatalist[i])
outfile.close()
def set_params(self, data_size, required_shares, max_shares):
assert required_shares <= max_shares
self.data_size = data_size
self.required_shares = required_shares
self.max_shares = max_shares
self.share_size = mathutil.div_ceil(data_size, required_shares)
self.last_share_padding = mathutil.pad_size(self.share_size, required_shares)
self.encoder = zfec.Encoder(required_shares, max_shares)
def encode_file(inf, cb, k, m, chunksize=4096):
"""
Read in the contents of inf, encode, and call cb with the results.
First, k "input blocks" will be read from inf, each input block being of
size chunksize. Then these k blocks will be encoded into m "result
blocks". Then cb will be invoked, passing a list of the m result blocks
as its first argument, and the length of the encoded data as its second
argument. (The length of the encoded data is always equal to k*chunksize,
until the last iteration, when the end of the file has been reached and
less than k*chunksize bytes could be read from the file.) This procedure
is iterated until the end of the file is reached, in which case the space
of the input blocks that is unused is filled with zeroes before encoding.
Note that the sequence passed in calls to cb() contains mutable array
objects in its first k elements whose contents will be overwritten when
the next segment is read from the input file. Therefore the
implementation of cb() has to either be finished with those first k arrays
before returning, or if it wants to keep the contents of those arrays for
subsequent use after it has returned then it must make a copy of them to
keep.
@param inf the file object from which to read the data
@param cb the callback to be invoked with the results
@param k the number of shares required to reconstruct the file
@param m the total number of shares created
@param chunksize how much data to read from inf for each of the k input
blocks
"""
enc = zfec.Encoder(k, m)
l = tuple([ array.array('c') for i in range(k) ])
indatasize = k*chunksize # will be reset to shorter upon EOF
eof = False
ZEROES=array.array('c', ['\x00'])*chunksize
while not eof:
# This loop body executes once per segment.
i = 0
while (i<len(l)):
# This loop body executes once per chunk.
a = l[i]
del a[:]
try:
a.fromfile(inf, chunksize)
i += 1
except EOFError:
eof = True
indatasize = i*chunksize + len(a)
# padding
a.fromstring("\x00" * (chunksize-len(a)))
i += 1
while (i<len(l)):
a = l[i]
a[:] = ZEROES
i += 1
res = enc.encode(l)
cb(res, indatasize)
def test_bad_args_construct_encoder(self):
try:
zfec.Encoder(-1, -1)
except zfec.Error as e:
assert "argument is required to be greater than or equal to 1" in str(
e), e
else:
self.fail(
"Should have gotten an exception from out-of-range arguments.")
try:
zfec.Encoder(1, 257)
except zfec.Error as e:
assert "argument is required to be less than or equal to 256" in str(
e), e
else:
self.fail(
"Should have gotten an exception from out-of-range arguments.")
def test_instantiate_encoder_no_args(self):
try:
e = zfec.Encoder()
except TypeError:
# Okay, so that's because we're required to pass constructor args.
pass
else:
# Oops, it should have raised an exception.
self.fail("Should have raised exception from incorrect arguments to constructor.")
def test_from_agl_py(self):
e = zfec.Encoder(3, 5)
b0 = '\x01'*8 ; b1 = '\x02'*8 ; b2 = '\x03'*8
# print "_from_py before encoding:"
# print "b0: %s, b1: %s, b2: %s" % tuple(base64.b16encode(x) for x in [b0, b1, b2])
b3, b4 = e.encode([b0, b1, b2], (3, 4))
# print "after encoding:"
# print "b3: %s, b4: %s" % tuple(base64.b16encode(x) for x in [b3, b4])
d = zfec.Decoder(3, 5)
r0, r1, r2 = d.decode((b2, b3, b4), (1, 2, 3))
def _h(k, m, ss):
encer = zfec.Encoder(k, m)
nums_and_blocks = list(enumerate(encer.encode(ss)))
assert isinstance(nums_and_blocks, list), nums_and_blocks
assert len(nums_and_blocks) == m, (len(nums_and_blocks), m,)
nums_and_blocks = random.sample(nums_and_blocks, k)
blocks = [ x[1] for x in nums_and_blocks ]
nums = [ x[0] for x in nums_and_blocks ]
decer = zfec.Decoder(k, m)
decoded = decer.decode(blocks, nums)
assert len(decoded) == len(ss), (len(decoded), len(ss),)
assert tuple([str(s) for s in decoded]) == tuple([str(s) for s in ss]), (tuple([ab(str(s)) for s in decoded]), tuple([ab(str(s)) for s in ss]),)
def main():
for i in range(2, 8):
N = 2**i
t = 2**(i - 2)
Threshold = ceil((N - t + 1) / 2.0)
zfecEncoder = zfec.Encoder(Threshold, N)
zfecDecoder = zfec.Decoder(Threshold, N)
for j in range(9, 12):
Tx = os.urandom((2**j) * 250)
start = time.time()
fragList = testEncoder(N, t, Tx, Threshold, zfecEncoder)
print N, t, 2**j, 'encode', time.time() - start
start = time.time()
zfecDecoder.decode(fragList[:Threshold], range(Threshold))
print N, t, 2**j, 'decode', time.time() - start
def _encode_obj(self, data):
"""Encode a single FEC object
Args:
data : Data to encode (bytes array)
Note:
Unlike method encode(), this method must take a data object that
fits within 256 chunks with overhead so that it fits into a single
FEC object. Furthermore, while encode() returns FEC packets,
this method returns FEC chunks.
Returns:
List of FEC-encoded chunks.
"""
assert (isinstance(data, bytes))
n_chunks = ceil(len(data) / CHUNK_SIZE)
n_overhead_chunks = ceil(self.overhead * n_chunks)
n_fec_chunks = n_chunks + n_overhead_chunks
assert (n_fec_chunks <= MAX_FEC_CHUNKS)
logger.debug("Original Chunks: {} / "
"Overhead Chunks: {} / "
"Total: {}".format(n_chunks, n_overhead_chunks,
n_fec_chunks))
# Split the given data array into chunks
chunks = []
for i_chunk in range(n_chunks):
# Byte range of the next chunk:
s_byte = i_chunk * CHUNK_SIZE # starting byte
e_byte = (i_chunk + 1) * CHUNK_SIZE # ending byte
chunk = data[s_byte:e_byte]
# The last chunk may need zero-padding
if (i_chunk + 1 == n_chunks and len(chunk) < CHUNK_SIZE):
chunk += bytes(CHUNK_SIZE - len(chunk))
assert (len(chunk) == CHUNK_SIZE)
chunks.append(chunk)
# Generate the corresponding FEC chunks
encoder = zfec.Encoder(n_chunks, n_fec_chunks)
return encoder.encode(chunks)
def encode(K, N, m):
"""Erasure encodes string ``m`` into ``N`` blocks, such that any ``K``
can reconstruct.
:param int K: K
:param int N: number of blocks to encode string ``m`` into.
:param int m: string to encode.
:return list: Erasure codes resulting from encoding ``m`` into
``N`` blocks using ``zfec`` lib.
"""
encoder = zfec.Encoder(K, N)
assert K <= 256 # TODO: Record this assumption!
# pad m to a multiple of K bytes
padlen = K - (len(m) % K)
m += padlen * chr(K - padlen)
step = len(m) / K
blocks = [m[i * step:(i + 1) * step] for i in range(K)]
stripes = encoder.encode(blocks)
return stripes
def ecEncode(src, dataChunks, parityChunks, setting):
'''Encode a file into data and parity chunks by erasure coding.'''
'''Use zfec library.'''
k = len(dataChunks)
n = k + len(parityChunks)
#Split src file into data chunks:
infile = open(src, 'rb')
data = infile.read()
infile.close()
totalchunk = len(dataChunks)
filesize = len(data)
if filesize == 0:
for i in range(totalchunk):
outfile = open(dataChunks[i], 'wb')
outfile.close()
else:
chunksize = filesize/totalchunk + 1
setting.chunksize = chunksize
chunkid = 0
for i in range(0, filesize+1, chunksize):
outfile = open(dataChunks[chunkid], 'wb')
startaddr = i
endaddr = i + chunksize
outfile.write(data[startaddr:endaddr])
if endaddr > filesize:
paddingsize = endaddr - (filesize)
for j in range(0, paddingsize):
outfile.write('\0')
outfile.close()
chunkid += 1
#Encode to generate parity chunks:
encoder = zfec.Encoder(k, n)
indatalist = []
for filename in dataChunks:
infile = open(filename, 'rb')
indatalist.append(infile.read())
infile.close()
outdatalist = encoder.encode(indatalist)
for i in range(n-k):
outfile = open(parityChunks[i],'wb')
outfile.write(outdatalist[k+i])
outfile.close()
def encode_file_stringy(inf, cb, k, m, chunksize=4096):
"""
Read in the contents of inf, encode, and call cb with the results.
First, k "input blocks" will be read from inf, each input block being of
size chunksize. Then these k blocks will be encoded into m "result
blocks". Then cb will be invoked, passing a list of the m result blocks
as its first argument, and the length of the encoded data as its second
argument. (The length of the encoded data is always equal to k*chunksize,
until the last iteration, when the end of the file has been reached and
less than k*chunksize bytes could be read from the file.) This procedure
is iterated until the end of the file is reached, in which case the part
of the input shares that is unused is filled with zeroes before encoding.
@param inf the file object from which to read the data
@param cb the callback to be invoked with the results
@param k the number of shares required to reconstruct the file
@param m the total number of shares created
@param chunksize how much data to read from inf for each of the k input
blocks
"""
enc = zfec.Encoder(k, m)
indatasize = k*chunksize # will be reset to shorter upon EOF
while indatasize == k*chunksize:
# This loop body executes once per segment.
i = 0
l = []
ZEROES = b'\x00'*chunksize
while i<k:
# This loop body executes once per chunk.
i += 1
l.append(inf.read(chunksize))
if len(l[-1]) < chunksize:
indatasize = i*chunksize + len(l[-1])
# padding
l[-1] = l[-1] + b"\x00" * (chunksize-len(l[-1]))
while i<k:
l.append(ZEROES)
i += 1
res = enc.encode(l)
cb(res, indatasize)
def encode(k, n, m):
"""Erasure encodes string ``m`` into ``n`` blocks, such that any ``k``
can reconstruct.
:param int k: k
:param int n: number of blocks to encode string ``m`` into.
:param bytes m: bytestring to encode.
:return list: Erasure codes resulting from encoding ``m`` into
``n`` blocks using ``zfec`` lib.
"""
try:
m = m.encode()
except AttributeError:
pass
encoder = zfec.Encoder(k, n)
assert k <= 256 # TODO: Record this assumption!
# pad m to a multiple of K bytes
padlen = k - (len(m) % k)
m += padlen * chr(k - padlen).encode()
step = len(m) // k
blocks = [m[i * step:(i + 1) * step] for i in range(k)]
stripes = encoder.encode(blocks)
return stripes
def multiSigBr(pid, N, t, msg, broadcast, receive, outputs, send):
# Since all the parties we have are symmetric, so I implement this function for N instances of A-cast as a whole
# Here msg is a set of transactions
assert (isinstance(outputs, list))
for i in outputs:
assert (isinstance(i, Queue))
keys = getECDSAKeys()
Threshold = N - 2 * t
Threshold2 = N - t
zfecEncoder = zfec.Encoder(Threshold, N)
zfecDecoder = zfec.Decoder(Threshold, N)
def merkleTree(strList, someHash=coolSHA256Hash):
# someHash is a mapping from a int to a int
treeLength = 2**ceil(math.log(len(strList)) / math.log(2))
mt = [0] * (treeLength * 2) # find a place to put our leaves
for i in range(len(strList)):
mt[i + treeLength] = someHash(
strList[i]
) # TODO: need to change strList[i] from a string to an integer here.
for i in range(treeLength - 1, 0, -1): # 1, 2, 3, ..., treeLength - 1
# mt[i] = someHash(''.join([chr(ord(a) ^ ord(b)) for a, b in zip(mt[i*2], mt[i*2+1])])) # XOR is commutative
mt[i] = someHash(mt[i * 2] +
mt[i * 2 + 1]) # concat is not commutative
return mt
def getMerkleBranch(index, mt):
res = []
t = index + (len(mt) >> 1)
while t > 1:
res.append(mt[t ^ 1]) # we are picking up the sibling
t /= 2
return res
def merkleVerify(val, rootHash, branch, someHash, index):
# index has information on whether we are facing a left sibling or a right sibling
tmp = someHash(val)
tindex = index
for br in branch:
tmp = someHash((tindex & 1) and br + tmp or tmp + br)
tindex >>= 1
if tmp != rootHash:
print "Verification failed with", someHash(
val), rootHash, branch, tmp == rootHash
return tmp == rootHash
def Listener():
opinions = [defaultdict(lambda: 0) for _ in range(N)]
rootHashes = dict()
readyCounter = [defaultdict(lambda: 0) for _ in range(N)]
signed = [False] * N
readySent = [False] * N
reconstDone = [False] * N
while True: # main loop
sender, msgBundle = receive()
if msgBundle[0] == 'i' and not signed[sender]:
if keys[sender].verify(
sha1hash(''.join([
msgBundle[1][0], msgBundle[1][1],
''.join(msgBundle[1][2])
])), msgBundle[2]):
assert isinstance(msgBundle[1], tuple)
if not merkleVerify(msgBundle[1][0], msgBundle[1][1],
msgBundle[1][2], coolSHA256Hash, pid):
continue
if sender in rootHashes:
if rootHashes[sender] != msgBundle[1][1]:
print "Cheating caught, exiting"
sys.exit(0)
else:
rootHashes[sender] = msgBundle[1][1]
newBundle = (sender, msgBundle[1][0], msgBundle[1][1],
msgBundle[1][2]
) # assert each frag has a length of step
broadcast(('e', newBundle, keys[pid].sign(
sha1hash(''.join([
str(newBundle[0]), newBundle[1], newBundle[2],
''.join(newBundle[3])
])))))
signed[sender] = True
else:
raise ECDSASignatureError()
elif msgBundle[0] == 'e':
if keys[sender].verify(
sha1hash(''.join([
str(msgBundle[1][0]), msgBundle[1][1],
msgBundle[1][2], ''.join(msgBundle[1][3])
])), msgBundle[2]):
originBundle = msgBundle[1]
if not merkleVerify(originBundle[1], originBundle[2],
originBundle[3], coolSHA256Hash,
sender):
continue
if originBundle[0] in rootHashes:
if rootHashes[originBundle[0]] != originBundle[2]:
print "Cheating caught, exiting"
sys.exit(0)
else:
rootHashes[originBundle[0]] = originBundle[2]
opinions[originBundle[0]][sender] = originBundle[
1] # We are going to move this part to kekeketktktktk
if len(opinions[originBundle[0]]
) >= Threshold2 and not readySent[originBundle[0]]:
readySent[originBundle[0]] = True
broadcast(
('r', originBundle[0],
originBundle[2])) # We are broadcasting its hash
else:
raise ECDSASignatureError()
elif msgBundle[0] == 'r':
readyCounter[msgBundle[1]][msgBundle[2]] += 1
tmp = readyCounter[msgBundle[1]][msgBundle[2]]
if tmp >= t + 1 and not readySent[msgBundle[1]]:
readySent[msgBundle[1]] = True
broadcast(('r', msgBundle[1], msgBundle[2]))
if tmp >= Threshold2 and not outputs[msgBundle[1]].full() and \
not reconstDone[msgBundle[1]] and len(opinions[msgBundle[1]]) >= Threshold:
reconstDone[msgBundle[1]] = True
if msgBundle[1] in rootHashes:
if rootHashes[msgBundle[1]] != msgBundle[2]:
print "Cheating caught, exiting"
sys.exit(0)
else:
rootHashes[msgBundle[1]] = msgBundle[2]
if opinions[msgBundle[1]].values()[0] == '':
reconstruction = ['']
else:
reconstruction = zfecDecoder.decode(
opinions[msgBundle[1]].values()[:Threshold],
opinions[msgBundle[1]].keys()[:Threshold]
) # We only take the first [Threshold] fragments
rawbuf = ''.join(reconstruction)
buf = rawbuf[:-ord(rawbuf[-1])]
# Check root hash
step = len(
buf
) / Threshold + 1 # len(buf) % Threshold == 0 and len(buf) / Threshold or (len(buf) / Threshold + 1)
assert step * Threshold - len(buf) < 256 # assumption
buf_ = buf.ljust(step * Threshold - 1,
'\xFF') + chr(step * Threshold - len(buf))
fragList = [
buf_[i * step:(i + 1) * step] for i in range(Threshold)
]
encodedFragList = zfecEncoder.encode(fragList)
mt = merkleTree(encodedFragList, coolSHA256Hash)
assert rootHashes[msgBundle[1]] == mt[
1] # full binary tree
if outputs[msgBundle[1]].empty():
outputs[msgBundle[1]].put(buf)
greenletPacker(Greenlet(Listener), 'multiSigBr.Listener',
(pid, N, t, msg, broadcast, receive, outputs)).start()
buf = msg # We already assumed the proposals are byte strings
step = len(
buf
) / Threshold + 1 # len(buf) % Threshold == 0 and len(buf) / Threshold or (len(buf) / Threshold + 1)
assert step * Threshold - len(buf) < 256 # assumption
buf = buf.ljust(step * Threshold - 1,
'\xFF') + chr(step * Threshold - len(buf))
fragList = [buf[i * step:(i + 1) * step] for i in range(Threshold)]
encodedFragList = zfecEncoder.encode(fragList)
mt = merkleTree(encodedFragList, coolSHA256Hash)
rootHash = mt[1] # full binary tree
for i in range(N):
mb = getMerkleBranch(
i, mt) # notice that index starts from 1 and pid starts from 0
newBundle = (encodedFragList[i], rootHash, mb)
send(i, ('i', newBundle, keys[pid].sign(
sha1hash(''.join(
[newBundle[0], newBundle[1], ''.join(newBundle[2])])))))
def test_bad_args_construct_encoder(self):
try:
zfec.Encoder(-1, -1)
except zfec.Error, e:
assert "argument is required to be greater than or equal to 1" in str(e), e
zfec.Decoder(3, 2)
except zfec.Error, e:
assert "first argument is required to be less than or equal to the second argument" in str(e), e
else:
self.fail("Should have gotten an exception from out-of-range arguments.")
def test_bad_args_construct_encoder(self):
try:
zfec.Encoder(-1, -1)
except zfec.Error, e:
assert "argument is required to be greater than or equal to 1" in str(e), e
else:
self.fail("Should have gotten an exception from out-of-range arguments.")
try:
zfec.Encoder(1, 257)
except zfec.Error, e:
assert "argument is required to be less than or equal to 256" in str(e), e
else:
self.fail("Should have gotten an exception from out-of-range arguments.")
def test_bad_args_dec(self):
decer = zfec.Decoder(2, 4)
try:
decer.decode(98, []) # first argument is not a sequence
except TypeError, e:
assert "First argument was not a sequence" in str(e), e
else:
self.fail("Should have gotten TypeError for wrong type of second argument.")
def __init__(self, k, m):
self.fec = zfec.Encoder(k, m)
import zfec
print(zfec.Encoder(1, 9).encode('hi'))
import os
import struct
from JumpScale import j
j.application.start("zfectest")
#@todo NOT WORKING, need to implement
import zfec
encoder = zfec.Encoder(10, 14)
filename = "/var/lib/libvirt/images/Ubuntu.img"
counter = 0
def encode(counter, series):
counter += 1
counterSeries = 0
seriesOut = encoder.encode(series)
for out in seriesOut:
counterSeries += 1
pathout = "out/Ubuntu.img.%s_%s" % (counter, counterSeries)
j.system.fs.writeFile(pathout, out)
print "%s_%s" % (counter, counterSeries)
return counter
size = 0
series = []
def __init__(self, k, m):
self.k = k
self.m = m
self.z_encoder = zfec.Encoder(k=k, m=m)
self.z_decoder = zfec.Decoder(k=k, m=m)
