def _decode_blocks(self, segnum, blocks):
start = now()
tail = (segnum == self.num_segments-1)
codec = self._codec
block_size = self.block_size
decoded_size = self.segment_size
if tail:
# account for the padding in the last segment
codec = CRSDecoder()
k, N = self._verifycap.needed_shares, self._verifycap.total_shares
codec.set_params(self.tail_segment_padded, k, N)
block_size = self.tail_block_size
decoded_size = self.tail_segment_padded
shares = []
shareids = []
for (shareid, share) in blocks.items():
assert len(share) == block_size
shareids.append(shareid)
shares.append(share)
del blocks
d = codec.decode(shares, shareids) # segment
del shares
def _process(buffers):
decodetime = now() - start
segment = b"".join(buffers)
assert len(segment) == decoded_size
del buffers
if tail:
segment = segment[:self.tail_segment_size]
self._download_status.add_misc_event("decode", start, now())
return (segment, decodetime)
d.addCallback(_process)
return d
def _parse_and_store_UEB(self, UEB_s):
# Note: the UEB contains needed_shares and total_shares. These are
# redundant and inferior (the filecap contains the authoritative
# values). However, because it is possible to encode the same file in
# multiple ways, and the encoders might choose (poorly) to use the
# same key for both (therefore getting the same SI), we might
# encounter shares for both types. The UEB hashes will be different,
# however, and we'll disregard the "other" encoding's shares as
# corrupted.
# therefore, we ignore d['total_shares'] and d['needed_shares'].
d = uri.unpack_extension(UEB_s)
log.msg(format="UEB=%(ueb)s, vcap=%(vcap)s",
ueb=repr(uri.unpack_extension_readable(UEB_s)),
vcap=self._verifycap.to_string(),
level=log.NOISY, parent=self._lp, umid="cVqZnA")
k, N = self._verifycap.needed_shares, self._verifycap.total_shares
self.segment_size = d['segment_size']
self._segsize_observers.fire(self.segment_size)
r = self._calculate_sizes(self.segment_size)
self.tail_segment_size = r["tail_segment_size"]
self.tail_segment_padded = r["tail_segment_padded"]
self.num_segments = r["num_segments"]
self.block_size = r["block_size"]
self.tail_block_size = r["tail_block_size"]
log.msg("actual sizes: %s" % (r,),
level=log.NOISY, parent=self._lp, umid="PY6P5Q")
if (self.segment_size == self.guessed_segment_size
and self.num_segments == self.guessed_num_segments):
log.msg("my guess was right!",
level=log.NOISY, parent=self._lp, umid="x340Ow")
else:
log.msg("my guess was wrong! Extra round trips for me.",
level=log.NOISY, parent=self._lp, umid="tb7RJw")
# zfec.Decode() instantiation is fast, but still, let's use the same
# codec instance for all but the last segment. 3-of-10 takes 15us on
# my laptop, 25-of-100 is 900us, 3-of-255 is 97us, 25-of-255 is
# 2.5ms, worst-case 254-of-255 is 9.3ms
self._codec = CRSDecoder()
self._codec.set_params(self.segment_size, k, N)
# Ciphertext hash tree root is mandatory, so that there is at most
# one ciphertext that matches this read-cap or verify-cap. The
# integrity check on the shares is not sufficient to prevent the
# original encoder from creating some shares of file A and other
# shares of file B. self.ciphertext_hash_tree was a guess before:
# this is where we create it for real.
self.ciphertext_hash_tree = IncompleteHashTree(self.num_segments)
self.ciphertext_hash_tree_leaves = self.num_segments
self.ciphertext_hash_tree.set_hashes({0: d['crypttext_root_hash']})
self.share_hash_tree.set_hashes({0: d['share_root_hash']})
def _decode_multiple(res):
log.msg("_decode_multiple")
# make sure we can re-use the decoder object
shares1 = random.sample(self.shares, required_shares)
sharesl1 = random.sample(list(zip(self.shares, self.shareids)), required_shares)
shares1 = [ x[0] for x in sharesl1 ]
shareids1 = [ x[1] for x in sharesl1 ]
sharesl2 = random.sample(list(zip(self.shares, self.shareids)), required_shares)
shares2 = [ x[0] for x in sharesl2 ]
shareids2 = [ x[1] for x in sharesl2 ]
dec = CRSDecoder()
dec.set_params(*params)
d1 = dec.decode(shares1, shareids1)
d1.addCallback(_check_data)
d1.addCallback(lambda res: dec.decode(shares2, shareids2))
d1.addCallback(_check_data)
return d1
def _decode(shares_and_shareids):
(shares, shareids) = shares_and_shareids
dec = CRSDecoder()
dec.set_params(*params)
d1 = dec.decode(shares, shareids)
return d1
def _decode((shares, shareids)):
dec = CRSDecoder()
dec.set_params(*params)
d1 = dec.decode(shares, shareids)
return d1
