def _update(self, data, offset): """ I update the mutable file version represented by this particular IMutableVersion by inserting the data in data at the offset offset. I return a Deferred that fires when this has been completed. """ new_size = data.get_size() + offset old_size = self.get_size() segment_size = self._version[3] num_old_segments = mathutil.div_ceil(old_size, segment_size) num_new_segments = mathutil.div_ceil(new_size, segment_size) log.msg("got %d old segments, %d new segments" % \ (num_old_segments, num_new_segments)) # We do a whole file re-encode if the file is an SDMF file. if self._version[2]: # version[2] == SDMF salt, which MDMF lacks log.msg("doing re-encode instead of in-place update") return self._do_modify_update(data, offset) # Otherwise, we can replace just the parts that are changing. log.msg("updating in place") d = self._do_update_update(data, offset) d.addCallback(self._decode_and_decrypt_segments, data, offset) d.addCallback(self._build_uploadable_and_finish, data, offset) return d
def _got_all_encoding_parameters(self, params): assert not self._codec k, happy, n, segsize = params self.required_shares = k self.min_happiness = happy self.num_shares = n self.segment_size = segsize self.log("got encoding parameters: %d/%d/%d %d" % (k, happy, n, segsize)) self.log("now setting up codec") assert self.segment_size % self.required_shares == 0 self.num_segments = mathutil.div_ceil(self.file_size, self.segment_size) self._codec = CRSEncoder() self._codec.set_params(self.segment_size, self.required_shares, self.num_shares) data = self.uri_extension_data data['codec_name'] = self._codec.get_encoder_type() data['codec_params'] = self._codec.get_serialized_params() data['size'] = self.file_size data['segment_size'] = self.segment_size self.share_size = mathutil.div_ceil(self.file_size, self.required_shares) data['num_segments'] = self.num_segments data['needed_shares'] = self.required_shares data['total_shares'] = self.num_shares # the "tail" is the last segment. This segment may or may not be # shorter than all other segments. We use the "tail codec" to handle # it. If the tail is short, we use a different codec instance. In # addition, the tail codec must be fed data which has been padded out # to the right size. tail_size = self.file_size % self.segment_size if not tail_size: tail_size = self.segment_size # the tail codec is responsible for encoding tail_size bytes padded_tail_size = mathutil.next_multiple(tail_size, self.required_shares) self._tail_codec = CRSEncoder() self._tail_codec.set_params(padded_tail_size, self.required_shares, self.num_shares) data['tail_codec_params'] = self._tail_codec.get_serialized_params()
def _calculate_sizes(self, segment_size): # segments of ciphertext size = self._verifycap.size k = self._verifycap.needed_shares # this assert matches the one in encode.py:127 inside # Encoded._got_all_encoding_parameters, where the UEB is constructed assert segment_size % k == 0 # the last segment is usually short. We don't store a whole segsize, # but we do pad the segment up to a multiple of k, because the # encoder requires that. tail_segment_size = size % segment_size if tail_segment_size == 0: tail_segment_size = segment_size padded = mathutil.next_multiple(tail_segment_size, k) tail_segment_padded = padded num_segments = mathutil.div_ceil(size, segment_size) # each segment is turned into N blocks. All but the last are of size # block_size, and the last is of size tail_block_size block_size = segment_size // k tail_block_size = tail_segment_padded // k return { "tail_segment_size": tail_segment_size, "tail_segment_padded": tail_segment_padded, "num_segments": num_segments, "block_size": block_size, "tail_block_size": tail_block_size }
def _calculate_sizes(self, segment_size): # segments of ciphertext size = self._verifycap.size k = self._verifycap.needed_shares # this assert matches the one in encode.py:127 inside # Encoded._got_all_encoding_parameters, where the UEB is constructed assert segment_size % k == 0 # the last segment is usually short. We don't store a whole segsize, # but we do pad the segment up to a multiple of k, because the # encoder requires that. tail_segment_size = size % segment_size if tail_segment_size == 0: tail_segment_size = segment_size padded = mathutil.next_multiple(tail_segment_size, k) tail_segment_padded = padded num_segments = mathutil.div_ceil(size, segment_size) # each segment is turned into N blocks. All but the last are of size # block_size, and the last is of size tail_block_size block_size = segment_size / k tail_block_size = tail_segment_padded / k return { "tail_segment_size": tail_segment_size, "tail_segment_padded": tail_segment_padded, "num_segments": num_segments, "block_size": block_size, "tail_block_size": tail_block_size, }
def _got_all_encoding_parameters(self, params): assert not self._codec k, happy, n, segsize = params self.required_shares = k self.servers_of_happiness = happy self.num_shares = n self.segment_size = segsize self.log("got encoding parameters: %d/%d/%d %d" % (k,happy,n, segsize)) self.log("now setting up codec") assert self.segment_size % self.required_shares == 0 self.num_segments = mathutil.div_ceil(self.file_size, self.segment_size) self._codec = CRSEncoder() self._codec.set_params(self.segment_size, self.required_shares, self.num_shares) data = self.uri_extension_data data['codec_name'] = self._codec.get_encoder_type() data['codec_params'] = self._codec.get_serialized_params() data['size'] = self.file_size data['segment_size'] = self.segment_size self.share_size = mathutil.div_ceil(self.file_size, self.required_shares) data['num_segments'] = self.num_segments data['needed_shares'] = self.required_shares data['total_shares'] = self.num_shares # the "tail" is the last segment. This segment may or may not be # shorter than all other segments. We use the "tail codec" to handle # it. If the tail is short, we use a different codec instance. In # addition, the tail codec must be fed data which has been padded out # to the right size. tail_size = self.file_size % self.segment_size if not tail_size: tail_size = self.segment_size # the tail codec is responsible for encoding tail_size bytes padded_tail_size = mathutil.next_multiple(tail_size, self.required_shares) self._tail_codec = CRSEncoder() self._tail_codec.set_params(padded_tail_size, self.required_shares, self.num_shares) data['tail_codec_params'] = self._tail_codec.get_serialized_params()
def set_params(self, data_size, required_shares, max_shares): assert required_shares <= max_shares self.data_size = data_size self.required_shares = required_shares self.max_shares = max_shares self.share_size = mathutil.div_ceil(data_size, required_shares) self.last_share_padding = mathutil.pad_size(self.share_size, required_shares) self.encoder = zfec.Encoder(required_shares, max_shares)
def set_params(self, data_size, required_shares, max_shares): self.data_size = data_size self.required_shares = required_shares self.max_shares = max_shares self.chunk_size = self.required_shares self.num_chunks = mathutil.div_ceil(self.data_size, self.chunk_size) self.share_size = self.num_chunks self.decoder = zfec.Decoder(self.required_shares, self.max_shares)
def remote_measure_peer_response_time(self): # I'd like to average together several pings, but I don't want this # phase to take more than 10 seconds. Expect worst-case latency to be # 300ms. results = {} sb = self.parent.get_storage_broker() everyone = sb.get_connected_servers() num_pings = int(mathutil.div_ceil(10, (len(everyone) * 0.3))) everyone = list(everyone) * num_pings d = self._do_one_ping(None, everyone, results) return d
def setup_encoding_parameters(self): segment_size = len(self.newdata) # this must be a multiple of self.required_shares segment_size = mathutil.next_multiple(segment_size, self.required_shares) self.segment_size = segment_size if segment_size: self.num_segments = mathutil.div_ceil(len(self.newdata), segment_size) else: self.num_segments = 0 assert self.num_segments in [0, 1,] # SDMF restrictions
def setup_encoding_parameters(self): segment_size = len(self.newdata) # this must be a multiple of self.required_shares segment_size = mathutil.next_multiple(segment_size, self.required_shares) self.segment_size = segment_size if segment_size: self.num_segments = mathutil.div_ceil(len(self.newdata), segment_size) else: self.num_segments = 0 assert self.num_segments in [ 0, 1, ] # SDMF restrictions
def test_odd_sizes(self): for j in range(2**6): lib = random.randrange(1, 2**8) numos = mathutil.div_ceil(lib, 8) bs = insecurerandstr(numos) # zero-out unused least-sig bits if lib%8: b=ord(bs[-1]) b = b >> (8 - (lib%8)) b = b << (8 - (lib%8)) bs = bs[:-1] + chr(b) asl = base62.b2a_l(bs, lib) assert len(asl) == base62.num_chars_that_this_many_octets_encode_to(numos) # the size of the base-62 encoding must be just right bs2l = base62.a2b_l(asl, lib) assert len(bs2l) == numos # the size of the result must be just right assert bs == bs2l
def _guess_offsets(self, verifycap, guessed_segment_size): self.guessed_segment_size = guessed_segment_size size = verifycap.size k = verifycap.needed_shares N = verifycap.total_shares r = self._node._calculate_sizes(guessed_segment_size) # num_segments, block_size/tail_block_size # guessed_segment_size/tail_segment_size/tail_segment_padded share_size = mathutil.div_ceil(size, k) # share_size is the amount of block data that will be put into each # share, summed over all segments. It does not include hashes, the # UEB, or other overhead. # use the upload-side code to get this as accurate as possible ht = IncompleteHashTree(N) num_share_hashes = len(ht.needed_hashes(0, include_leaf=True)) wbp = make_write_bucket_proxy(None, None, share_size, r["block_size"], r["num_segments"], num_share_hashes, 0) self._fieldsize = wbp.fieldsize self._fieldstruct = wbp.fieldstruct self.guessed_offsets = wbp._offsets
def calc(filesize, params=(3, 7, 10), segsize=DEFAULT_MAX_SEGMENT_SIZE): num_shares = params[2] if filesize <= upload.Uploader.URI_LIT_SIZE_THRESHOLD: urisize = len(uri.LiteralFileURI("A" * filesize).to_string()) sharesize = 0 sharespace = 0 else: u = upload.FileUploader(None) # XXX changed u.set_params(params) # unfortunately, Encoder doesn't currently lend itself to answering # this question without measuring a filesize, so we have to give it a # fake one data = BigFakeString(filesize) u.set_filehandle(data) u.set_encryption_key("a" * 16) sharesize, blocksize = u.setup_encoder() # how much overhead? # 0x20 bytes of offsets # 0x04 bytes of extension length # 0x1ad bytes of extension (=429) # total is 465 bytes num_segments = mathutil.div_ceil(filesize, segsize) num_share_hashes = int( math.log(mathutil.next_power_of_k(num_shares, 2), 2)) + 1 sharesize = storage.allocated_size(sharesize, num_segments, num_share_hashes, 429) sharespace = num_shares * roundup(sharesize) urisize = len( uri.pack_uri(storage_index="a" * 32, key="a" * 16, uri_extension_hash="a" * 32, needed_shares=params[0], total_shares=params[2], size=filesize)) return urisize, sharesize, sharespace
def calc(filesize, params=(3,7,10), segsize=DEFAULT_MAX_SEGMENT_SIZE): num_shares = params[2] if filesize <= upload.Uploader.URI_LIT_SIZE_THRESHOLD: urisize = len(uri.LiteralFileURI("A"*filesize).to_string()) sharesize = 0 sharespace = 0 else: u = upload.FileUploader(None) # XXX changed u.set_params(params) # unfortunately, Encoder doesn't currently lend itself to answering # this question without measuring a filesize, so we have to give it a # fake one data = BigFakeString(filesize) u.set_filehandle(data) u.set_encryption_key("a"*16) sharesize, blocksize = u.setup_encoder() # how much overhead? # 0x20 bytes of offsets # 0x04 bytes of extension length # 0x1ad bytes of extension (=429) # total is 465 bytes num_segments = mathutil.div_ceil(filesize, segsize) num_share_hashes = int(math.log(mathutil.next_power_of_k(num_shares, 2), 2)) + 1 sharesize = storage.allocated_size(sharesize, num_segments, num_share_hashes, 429) sharespace = num_shares * roundup(sharesize) urisize = len(uri.pack_uri(storage_index="a"*32, key="a"*16, uri_extension_hash="a"*32, needed_shares=params[0], total_shares=params[2], size=filesize)) return urisize, sharesize, sharespace
def _setup_encoding_parameters(self): """ I set up the encoding parameters, including k, n, the number of segments associated with this file, and the segment decoders. """ (seqnum, root_hash, IV, segsize, datalength, k, n, known_prefix, offsets_tuple) = self.verinfo self._required_shares = k self._total_shares = n self._segment_size = segsize #self._data_length = datalength # set during __init__() if not IV: self._version = MDMF_VERSION else: self._version = SDMF_VERSION if datalength and segsize: self._num_segments = mathutil.div_ceil(datalength, segsize) self._tail_data_size = datalength % segsize else: self._num_segments = 0 self._tail_data_size = 0 self._segment_decoder = codec.CRSDecoder() self._segment_decoder.set_params(segsize, k, n) if not self._tail_data_size: self._tail_data_size = segsize self._tail_segment_size = mathutil.next_multiple(self._tail_data_size, self._required_shares) if self._tail_segment_size == self._segment_size: self._tail_decoder = self._segment_decoder else: self._tail_decoder = codec.CRSDecoder() self._tail_decoder.set_params(self._tail_segment_size, self._required_shares, self._total_shares) self.log("got encoding parameters: " "k: %d " "n: %d " "%d segments of %d bytes each (%d byte tail segment)" % \ (k, n, self._num_segments, self._segment_size, self._tail_segment_size)) # Our last task is to tell the downloader where to start and # where to stop. We use three parameters for that: # - self._start_segment: the segment that we need to start # downloading from. # - self._current_segment: the next segment that we need to # download. # - self._last_segment: The last segment that we were asked to # download. # # We say that the download is complete when # self._current_segment > self._last_segment. We use # self._start_segment and self._last_segment to know when to # strip things off of segments, and how much to strip. if self._offset: self.log("got offset: %d" % self._offset) # our start segment is the first segment containing the # offset we were given. start = self._offset // self._segment_size _assert(start <= self._num_segments, start=start, num_segments=self._num_segments, offset=self._offset, segment_size=self._segment_size) self._start_segment = start self.log("got start segment: %d" % self._start_segment) else: self._start_segment = 0 # We might want to read only part of the file, and need to figure out # where to stop reading. Our end segment is the last segment # containing part of the segment that we were asked to read. _assert(self._read_length > 0, self._read_length) end_data = self._offset + self._read_length # We don't actually need to read the byte at end_data, but the one # before it. end = (end_data - 1) // self._segment_size _assert(0 <= end < self._num_segments, end=end, num_segments=self._num_segments, end_data=end_data, offset=self._offset, read_length=self._read_length, segment_size=self._segment_size) self._last_segment = end self.log("got end segment: %d" % self._last_segment) self._current_segment = self._start_segment
def roundup(size, blocksize=4096): return blocksize * mathutil.div_ceil(size, blocksize)
def do_test(self, size, required_shares, max_shares, fewer_shares=None): data0s = [os.urandom(mathutil.div_ceil(size, required_shares)) for i in range(required_shares)] enc = CRSEncoder() enc.set_params(size, required_shares, max_shares) params = enc.get_params() assert params == (size, required_shares, max_shares) log.msg("params: %s" % (params,)) d = enc.encode(data0s) def _done_encoding_all(shares_and_shareids): (shares, shareids) = shares_and_shareids self.failUnlessEqual(len(shares), max_shares) self.shares = shares self.shareids = shareids d.addCallback(_done_encoding_all) if fewer_shares is not None: # also validate that the desired_shareids= parameter works desired_shareids = random.sample(range(max_shares), fewer_shares) d.addCallback(lambda res: enc.encode(data0s, desired_shareids)) def _check_fewer_shares(some_shares_and_their_shareids): (some_shares, their_shareids) = some_shares_and_their_shareids self.failUnlessEqual(tuple(their_shareids), tuple(desired_shareids)) d.addCallback(_check_fewer_shares) def _decode(shares_and_shareids): (shares, shareids) = shares_and_shareids dec = CRSDecoder() dec.set_params(*params) d1 = dec.decode(shares, shareids) return d1 def _check_data(decoded_shares): self.failUnlessEqual(len(''.join(decoded_shares)), len(''.join(data0s))) self.failUnlessEqual(len(decoded_shares), len(data0s)) for (i, (x, y)) in enumerate(zip(data0s, decoded_shares)): self.failUnlessEqual(x, y, "%s: %r != %r.... first share was %r" % (str(i), x, y, data0s[0],)) self.failUnless(''.join(decoded_shares) == ''.join(data0s), "%s" % ("???",)) # 0data0sclipped = tuple(data0s) # data0sclipped[-1] = # self.failUnless(tuple(decoded_shares) == tuple(data0s)) def _decode_some(res): log.msg("_decode_some") # decode with a minimal subset of the shares some_shares = self.shares[:required_shares] some_shareids = self.shareids[:required_shares] return _decode((some_shares, some_shareids)) d.addCallback(_decode_some) d.addCallback(_check_data) def _decode_some_random(res): log.msg("_decode_some_random") # use a randomly-selected minimal subset l = random.sample(zip(self.shares, self.shareids), required_shares) some_shares = [ x[0] for x in l ] some_shareids = [ x[1] for x in l ] return _decode((some_shares, some_shareids)) d.addCallback(_decode_some_random) d.addCallback(_check_data) def _decode_multiple(res): log.msg("_decode_multiple") # make sure we can re-use the decoder object shares1 = random.sample(self.shares, required_shares) sharesl1 = random.sample(zip(self.shares, self.shareids), required_shares) shares1 = [ x[0] for x in sharesl1 ] shareids1 = [ x[1] for x in sharesl1 ] sharesl2 = random.sample(zip(self.shares, self.shareids), required_shares) shares2 = [ x[0] for x in sharesl2 ] shareids2 = [ x[1] for x in sharesl2 ] dec = CRSDecoder() dec.set_params(*params) d1 = dec.decode(shares1, shareids1) d1.addCallback(_check_data) d1.addCallback(lambda res: dec.decode(shares2, shareids2)) d1.addCallback(_check_data) return d1 d.addCallback(_decode_multiple) return d
def _parse_and_validate(self, data): self.share_size = mathutil.div_ceil(self._verifycap.size, self._verifycap.needed_shares) d = uri.unpack_extension(data) # There are several kinds of things that can be found in a UEB. # First, things that we really need to learn from the UEB in order to # do this download. Next: things which are optional but not redundant # -- if they are present in the UEB they will get used. Next, things # that are optional and redundant. These things are required to be # consistent: they don't have to be in the UEB, but if they are in # the UEB then they will be checked for consistency with the # already-known facts, and if they are inconsistent then an exception # will be raised. These things aren't actually used -- they are just # tested for consistency and ignored. Finally: things which are # deprecated -- they ought not be in the UEB at all, and if they are # present then a warning will be logged but they are otherwise # ignored. # First, things that we really need to learn from the UEB: # segment_size, crypttext_root_hash, and share_root_hash. self.segment_size = d['segment_size'] self.block_size = mathutil.div_ceil(self.segment_size, self._verifycap.needed_shares) self.num_segments = mathutil.div_ceil(self._verifycap.size, self.segment_size) self.tail_data_size = self._verifycap.size % self.segment_size if not self.tail_data_size: self.tail_data_size = self.segment_size # padding for erasure code self.tail_segment_size = mathutil.next_multiple(self.tail_data_size, self._verifycap.needed_shares) # Ciphertext hash tree root is mandatory, so that there is at most # one ciphertext that matches this read-cap or verify-cap. The # integrity check on the shares is not sufficient to prevent the # original encoder from creating some shares of file A and other # shares of file B. self.crypttext_root_hash = d['crypttext_root_hash'] self.share_root_hash = d['share_root_hash'] # Next: things that are optional and not redundant: crypttext_hash if d.has_key('crypttext_hash'): self.crypttext_hash = d['crypttext_hash'] if len(self.crypttext_hash) != CRYPTO_VAL_SIZE: raise BadURIExtension('crypttext_hash is required to be hashutil.CRYPTO_VAL_SIZE bytes, not %s bytes' % (len(self.crypttext_hash),)) # Next: things that are optional, redundant, and required to be # consistent: codec_name, codec_params, tail_codec_params, # num_segments, size, needed_shares, total_shares if d.has_key('codec_name'): if d['codec_name'] != "crs": raise UnsupportedErasureCodec(d['codec_name']) if d.has_key('codec_params'): ucpss, ucpns, ucpts = codec.parse_params(d['codec_params']) if ucpss != self.segment_size: raise BadURIExtension("inconsistent erasure code params: " "ucpss: %s != self.segment_size: %s" % (ucpss, self.segment_size)) if ucpns != self._verifycap.needed_shares: raise BadURIExtension("inconsistent erasure code params: ucpns: %s != " "self._verifycap.needed_shares: %s" % (ucpns, self._verifycap.needed_shares)) if ucpts != self._verifycap.total_shares: raise BadURIExtension("inconsistent erasure code params: ucpts: %s != " "self._verifycap.total_shares: %s" % (ucpts, self._verifycap.total_shares)) if d.has_key('tail_codec_params'): utcpss, utcpns, utcpts = codec.parse_params(d['tail_codec_params']) if utcpss != self.tail_segment_size: raise BadURIExtension("inconsistent erasure code params: utcpss: %s != " "self.tail_segment_size: %s, self._verifycap.size: %s, " "self.segment_size: %s, self._verifycap.needed_shares: %s" % (utcpss, self.tail_segment_size, self._verifycap.size, self.segment_size, self._verifycap.needed_shares)) if utcpns != self._verifycap.needed_shares: raise BadURIExtension("inconsistent erasure code params: utcpns: %s != " "self._verifycap.needed_shares: %s" % (utcpns, self._verifycap.needed_shares)) if utcpts != self._verifycap.total_shares: raise BadURIExtension("inconsistent erasure code params: utcpts: %s != " "self._verifycap.total_shares: %s" % (utcpts, self._verifycap.total_shares)) if d.has_key('num_segments'): if d['num_segments'] != self.num_segments: raise BadURIExtension("inconsistent num_segments: size: %s, " "segment_size: %s, computed_num_segments: %s, " "ueb_num_segments: %s" % (self._verifycap.size, self.segment_size, self.num_segments, d['num_segments'])) if d.has_key('size'): if d['size'] != self._verifycap.size: raise BadURIExtension("inconsistent size: URI size: %s, UEB size: %s" % (self._verifycap.size, d['size'])) if d.has_key('needed_shares'): if d['needed_shares'] != self._verifycap.needed_shares: raise BadURIExtension("inconsistent needed shares: URI needed shares: %s, UEB " "needed shares: %s" % (self._verifycap.total_shares, d['needed_shares'])) if d.has_key('total_shares'): if d['total_shares'] != self._verifycap.total_shares: raise BadURIExtension("inconsistent total shares: URI total shares: %s, UEB " "total shares: %s" % (self._verifycap.total_shares, d['total_shares'])) # Finally, things that are deprecated and ignored: plaintext_hash, # plaintext_root_hash if d.get('plaintext_hash'): log.msg("Found plaintext_hash in UEB. This field is deprecated for security reasons " "and is no longer used. Ignoring. %s" % (self,)) if d.get('plaintext_root_hash'): log.msg("Found plaintext_root_hash in UEB. This field is deprecated for security " "reasons and is no longer used. Ignoring. %s" % (self,)) return self
def _get_share_size(self): share_size = mathutil.div_ceil(self.file_size, self.required_shares) overhead = self._compute_overhead() return share_size + overhead
def _parse_and_validate(self, data): self.share_size = mathutil.div_ceil(self._verifycap.size, self._verifycap.needed_shares) d = uri.unpack_extension(data) # There are several kinds of things that can be found in a UEB. # First, things that we really need to learn from the UEB in order to # do this download. Next: things which are optional but not redundant # -- if they are present in the UEB they will get used. Next, things # that are optional and redundant. These things are required to be # consistent: they don't have to be in the UEB, but if they are in # the UEB then they will be checked for consistency with the # already-known facts, and if they are inconsistent then an exception # will be raised. These things aren't actually used -- they are just # tested for consistency and ignored. Finally: things which are # deprecated -- they ought not be in the UEB at all, and if they are # present then a warning will be logged but they are otherwise # ignored. # First, things that we really need to learn from the UEB: # segment_size, crypttext_root_hash, and share_root_hash. self.segment_size = d['segment_size'] self.block_size = mathutil.div_ceil(self.segment_size, self._verifycap.needed_shares) self.num_segments = mathutil.div_ceil(self._verifycap.size, self.segment_size) self.tail_data_size = self._verifycap.size % self.segment_size if not self.tail_data_size: self.tail_data_size = self.segment_size # padding for erasure code self.tail_segment_size = mathutil.next_multiple( self.tail_data_size, self._verifycap.needed_shares) # Ciphertext hash tree root is mandatory, so that there is at most # one ciphertext that matches this read-cap or verify-cap. The # integrity check on the shares is not sufficient to prevent the # original encoder from creating some shares of file A and other # shares of file B. self.crypttext_root_hash = d['crypttext_root_hash'] self.share_root_hash = d['share_root_hash'] # Next: things that are optional and not redundant: crypttext_hash if d.has_key('crypttext_hash'): self.crypttext_hash = d['crypttext_hash'] if len(self.crypttext_hash) != CRYPTO_VAL_SIZE: raise BadURIExtension( 'crypttext_hash is required to be hashutil.CRYPTO_VAL_SIZE bytes, not %s bytes' % (len(self.crypttext_hash), )) # Next: things that are optional, redundant, and required to be # consistent: codec_name, codec_params, tail_codec_params, # num_segments, size, needed_shares, total_shares if d.has_key('codec_name'): if d['codec_name'] != "crs": raise UnsupportedErasureCodec(d['codec_name']) if d.has_key('codec_params'): ucpss, ucpns, ucpts = codec.parse_params(d['codec_params']) if ucpss != self.segment_size: raise BadURIExtension("inconsistent erasure code params: " "ucpss: %s != self.segment_size: %s" % (ucpss, self.segment_size)) if ucpns != self._verifycap.needed_shares: raise BadURIExtension( "inconsistent erasure code params: ucpns: %s != " "self._verifycap.needed_shares: %s" % (ucpns, self._verifycap.needed_shares)) if ucpts != self._verifycap.total_shares: raise BadURIExtension( "inconsistent erasure code params: ucpts: %s != " "self._verifycap.total_shares: %s" % (ucpts, self._verifycap.total_shares)) if d.has_key('tail_codec_params'): utcpss, utcpns, utcpts = codec.parse_params(d['tail_codec_params']) if utcpss != self.tail_segment_size: raise BadURIExtension( "inconsistent erasure code params: utcpss: %s != " "self.tail_segment_size: %s, self._verifycap.size: %s, " "self.segment_size: %s, self._verifycap.needed_shares: %s" % (utcpss, self.tail_segment_size, self._verifycap.size, self.segment_size, self._verifycap.needed_shares)) if utcpns != self._verifycap.needed_shares: raise BadURIExtension( "inconsistent erasure code params: utcpns: %s != " "self._verifycap.needed_shares: %s" % (utcpns, self._verifycap.needed_shares)) if utcpts != self._verifycap.total_shares: raise BadURIExtension( "inconsistent erasure code params: utcpts: %s != " "self._verifycap.total_shares: %s" % (utcpts, self._verifycap.total_shares)) if d.has_key('num_segments'): if d['num_segments'] != self.num_segments: raise BadURIExtension( "inconsistent num_segments: size: %s, " "segment_size: %s, computed_num_segments: %s, " "ueb_num_segments: %s" % (self._verifycap.size, self.segment_size, self.num_segments, d['num_segments'])) if d.has_key('size'): if d['size'] != self._verifycap.size: raise BadURIExtension( "inconsistent size: URI size: %s, UEB size: %s" % (self._verifycap.size, d['size'])) if d.has_key('needed_shares'): if d['needed_shares'] != self._verifycap.needed_shares: raise BadURIExtension( "inconsistent needed shares: URI needed shares: %s, UEB " "needed shares: %s" % (self._verifycap.total_shares, d['needed_shares'])) if d.has_key('total_shares'): if d['total_shares'] != self._verifycap.total_shares: raise BadURIExtension( "inconsistent total shares: URI total shares: %s, UEB " "total shares: %s" % (self._verifycap.total_shares, d['total_shares'])) # Finally, things that are deprecated and ignored: plaintext_hash, # plaintext_root_hash if d.get('plaintext_hash'): log.msg( "Found plaintext_hash in UEB. This field is deprecated for security reasons " "and is no longer used. Ignoring. %s" % (self, )) if d.get('plaintext_root_hash'): log.msg( "Found plaintext_root_hash in UEB. This field is deprecated for security " "reasons and is no longer used. Ignoring. %s" % (self, )) return self
def _setup_encoding_parameters(self): """ I set up the encoding parameters, including k, n, the number of segments associated with this file, and the segment decoders. """ (seqnum, root_hash, IV, segsize, datalength, k, n, known_prefix, offsets_tuple) = self.verinfo self._required_shares = k self._total_shares = n self._segment_size = segsize self._data_length = datalength if not IV: self._version = MDMF_VERSION else: self._version = SDMF_VERSION if datalength and segsize: self._num_segments = mathutil.div_ceil(datalength, segsize) self._tail_data_size = datalength % segsize else: self._num_segments = 0 self._tail_data_size = 0 self._segment_decoder = codec.CRSDecoder() self._segment_decoder.set_params(segsize, k, n) if not self._tail_data_size: self._tail_data_size = segsize self._tail_segment_size = mathutil.next_multiple(self._tail_data_size, self._required_shares) if self._tail_segment_size == self._segment_size: self._tail_decoder = self._segment_decoder else: self._tail_decoder = codec.CRSDecoder() self._tail_decoder.set_params(self._tail_segment_size, self._required_shares, self._total_shares) self.log("got encoding parameters: " "k: %d " "n: %d " "%d segments of %d bytes each (%d byte tail segment)" % \ (k, n, self._num_segments, self._segment_size, self._tail_segment_size)) if self._block_hash_trees is not None: for i in xrange(self._total_shares): # So we don't have to do this later. self._block_hash_trees[i] = hashtree.IncompleteHashTree(self._num_segments) # Our last task is to tell the downloader where to start and # where to stop. We use three parameters for that: # - self._start_segment: the segment that we need to start # downloading from. # - self._current_segment: the next segment that we need to # download. # - self._last_segment: The last segment that we were asked to # download. # # We say that the download is complete when # self._current_segment > self._last_segment. We use # self._start_segment and self._last_segment to know when to # strip things off of segments, and how much to strip. if self._offset: self.log("got offset: %d" % self._offset) # our start segment is the first segment containing the # offset we were given. start = self._offset // self._segment_size assert start < self._num_segments self._start_segment = start self.log("got start segment: %d" % self._start_segment) else: self._start_segment = 0 # If self._read_length is None, then we want to read the whole # file. Otherwise, we want to read only part of the file, and # need to figure out where to stop reading. if self._read_length is not None: # our end segment is the last segment containing part of the # segment that we were asked to read. self.log("got read length %d" % self._read_length) if self._read_length != 0: end_data = self._offset + self._read_length # We don't actually need to read the byte at end_data, # but the one before it. end = (end_data - 1) // self._segment_size assert end < self._num_segments self._last_segment = end else: self._last_segment = self._start_segment self.log("got end segment: %d" % self._last_segment) else: self._last_segment = self._num_segments - 1 self._current_segment = self._start_segment
def do_forms(self, getarg): filled = getarg("filled", bool) def get_and_set(name, options, default=None, astype=int): current_value = getarg(name, astype) i_select = T.select(name=name) for (count, description) in options: count = astype(count) if ((current_value is not None and count == current_value) or (current_value is None and count == default)): o = T.option(value=str(count), selected="true")[description] else: o = T.option(value=str(count))[description] i_select = i_select[o] if current_value is None: current_value = default return current_value, i_select sections = {} def add_input(section, text, entry): if section not in sections: sections[section] = [] sections[section].extend([T.div[text, ": ", entry], "\n"]) def add_output(section, entry): if section not in sections: sections[section] = [] sections[section].extend([entry, "\n"]) def build_section(section): return T.fieldset[T.legend[section], sections[section]] def number(value, suffix=""): scaling = 1 if value < 1: fmt = "%1.2g%s" elif value < 100: fmt = "%.1f%s" elif value < 1000: fmt = "%d%s" elif value < 1e6: fmt = "%.2fk%s" scaling = 1e3 elif value < 1e9: fmt = "%.2fM%s" scaling = 1e6 elif value < 1e12: fmt = "%.2fG%s" scaling = 1e9 elif value < 1e15: fmt = "%.2fT%s" scaling = 1e12 elif value < 1e18: fmt = "%.2fP%s" scaling = 1e15 else: fmt = "huge! %g%s" return fmt % (value / scaling, suffix) user_counts = [ (5, "5 users"), (50, "50 users"), (200, "200 users"), (1000, "1k users"), (10000, "10k users"), (50000, "50k users"), (100000, "100k users"), (500000, "500k users"), (1000000, "1M users"), ] num_users, i_num_users = get_and_set("num_users", user_counts, 50000) add_input("Users", "How many users are on this network?", i_num_users) files_per_user_counts = [ (100, "100 files"), (1000, "1k files"), (10000, "10k files"), (100000, "100k files"), (1e6, "1M files"), ] files_per_user, i_files_per_user = get_and_set("files_per_user", files_per_user_counts, 1000) add_input("Users", "How many files for each user? (avg)", i_files_per_user) space_per_user_sizes = [ (1e6, "1MB"), (10e6, "10MB"), (100e6, "100MB"), (200e6, "200MB"), (1e9, "1GB"), (2e9, "2GB"), (5e9, "5GB"), (10e9, "10GB"), (100e9, "100GB"), (1e12, "1TB"), ] # current allmydata average utilization 127MB per user space_per_user, i_space_per_user = get_and_set("space_per_user", space_per_user_sizes, 200e6) add_input("Users", "How much data for each user? (avg)", i_space_per_user) sharing_ratios = [ (1.0, "1.0x"), (1.1, "1.1x"), (2.0, "2.0x"), ] sharing_ratio, i_sharing_ratio = get_and_set("sharing_ratio", sharing_ratios, 1.0, float) add_input( "Users", "What is the sharing ratio? (1.0x is no-sharing and" " no convergence)", i_sharing_ratio) # Encoding parameters encoding_choices = [ ("3-of-10-5", "3.3x (3-of-10, repair below 5)"), ("3-of-10-8", "3.3x (3-of-10, repair below 8)"), ("5-of-10-7", "2x (5-of-10, repair below 7)"), ("8-of-10-9", "1.25x (8-of-10, repair below 9)"), ("27-of-30-28", "1.1x (27-of-30, repair below 28"), ("25-of-100-50", "4x (25-of-100, repair below 50)"), ] encoding_parameters, i_encoding_parameters = \ get_and_set("encoding_parameters", encoding_choices, "3-of-10-5", str) encoding_pieces = encoding_parameters.split("-") k = int(encoding_pieces[0]) assert encoding_pieces[1] == "of" n = int(encoding_pieces[2]) # we repair the file when the number of available shares drops below # this value repair_threshold = int(encoding_pieces[3]) add_input("Servers", "What are the default encoding parameters?", i_encoding_parameters) # Server info num_server_choices = [ (5, "5 servers"), (10, "10 servers"), (15, "15 servers"), (30, "30 servers"), (50, "50 servers"), (100, "100 servers"), (200, "200 servers"), (300, "300 servers"), (500, "500 servers"), (1000, "1k servers"), (2000, "2k servers"), (5000, "5k servers"), (10e3, "10k servers"), (100e3, "100k servers"), (1e6, "1M servers"), ] num_servers, i_num_servers = \ get_and_set("num_servers", num_server_choices, 30, int) add_input("Servers", "How many servers are there?", i_num_servers) # availability is measured in dBA = -dBF, where 0dBF is 100% failure, # 10dBF is 10% failure, 20dBF is 1% failure, etc server_dBA_choices = [ (10, "90% [10dBA] (2.4hr/day)"), (13, "95% [13dBA] (1.2hr/day)"), (20, "99% [20dBA] (14min/day or 3.5days/year)"), (23, "99.5% [23dBA] (7min/day or 1.75days/year)"), (30, "99.9% [30dBA] (87sec/day or 9hours/year)"), (40, "99.99% [40dBA] (60sec/week or 53min/year)"), (50, "99.999% [50dBA] (5min per year)"), ] server_dBA, i_server_availability = \ get_and_set("server_availability", server_dBA_choices, 20, int) add_input("Servers", "What is the server availability?", i_server_availability) drive_MTBF_choices = [ (40, "40,000 Hours"), ] drive_MTBF, i_drive_MTBF = \ get_and_set("drive_MTBF", drive_MTBF_choices, 40, int) add_input("Drives", "What is the hard drive MTBF?", i_drive_MTBF) # http://www.tgdaily.com/content/view/30990/113/ # http://labs.google.com/papers/disk_failures.pdf # google sees: # 1.7% of the drives they replaced were 0-1 years old # 8% of the drives they repalced were 1-2 years old # 8.6% were 2-3 years old # 6% were 3-4 years old, about 8% were 4-5 years old drive_size_choices = [ (100, "100 GB"), (250, "250 GB"), (500, "500 GB"), (750, "750 GB"), ] drive_size, i_drive_size = \ get_and_set("drive_size", drive_size_choices, 750, int) drive_size = drive_size * 1e9 add_input("Drives", "What is the capacity of each hard drive?", i_drive_size) drive_failure_model_choices = [ ("E", "Exponential"), ("U", "Uniform"), ] drive_failure_model, i_drive_failure_model = \ get_and_set("drive_failure_model", drive_failure_model_choices, "E", str) add_input("Drives", "How should we model drive failures?", i_drive_failure_model) # drive_failure_rate is in failures per second if drive_failure_model == "E": drive_failure_rate = 1.0 / (drive_MTBF * 1000 * 3600) else: drive_failure_rate = 0.5 / (drive_MTBF * 1000 * 3600) # deletion/gc/ownership mode ownership_choices = [ ("A", "no deletion, no gc, no owners"), ("B", "deletion, no gc, no owners"), ("C", "deletion, share timers, no owners"), ("D", "deletion, no gc, yes owners"), ("E", "deletion, owner timers"), ] ownership_mode, i_ownership_mode = \ get_and_set("ownership_mode", ownership_choices, "A", str) add_input("Servers", "What is the ownership mode?", i_ownership_mode) # client access behavior access_rates = [ (1, "one file per day"), (10, "10 files per day"), (100, "100 files per day"), (1000, "1k files per day"), (10e3, "10k files per day"), (100e3, "100k files per day"), ] download_files_per_day, i_download_rate = \ get_and_set("download_rate", access_rates, 100, int) add_input("Users", "How many files are downloaded per day?", i_download_rate) download_rate = 1.0 * download_files_per_day / (24 * 60 * 60) upload_files_per_day, i_upload_rate = \ get_and_set("upload_rate", access_rates, 10, int) add_input("Users", "How many files are uploaded per day?", i_upload_rate) upload_rate = 1.0 * upload_files_per_day / (24 * 60 * 60) delete_files_per_day, i_delete_rate = \ get_and_set("delete_rate", access_rates, 10, int) add_input("Users", "How many files are deleted per day?", i_delete_rate) delete_rate = 1.0 * delete_files_per_day / (24 * 60 * 60) # the value is in days lease_timers = [ (1, "one refresh per day"), (7, "one refresh per week"), ] lease_timer, i_lease = \ get_and_set("lease_timer", lease_timers, 7, int) add_input( "Users", "How frequently do clients refresh files or accounts? " "(if necessary)", i_lease) seconds_per_lease = 24 * 60 * 60 * lease_timer check_timer_choices = [ (1, "every week"), (4, "every month"), (8, "every two months"), (16, "every four months"), ] check_timer, i_check_timer = \ get_and_set("check_timer", check_timer_choices, 4, int) add_input("Users", "How frequently should we check on each file?", i_check_timer) file_check_interval = check_timer * 7 * 24 * 3600 if filled: add_output("Users", T.div["Total users: %s" % number(num_users)]) add_output("Users", T.div["Files per user: %s" % number(files_per_user)]) file_size = 1.0 * space_per_user / files_per_user add_output("Users", T.div["Average file size: ", number(file_size)]) total_files = num_users * files_per_user / sharing_ratio add_output( "Grid", T.div["Total number of files in grid: ", number(total_files)]) total_space = num_users * space_per_user / sharing_ratio add_output( "Grid", T.div["Total volume of plaintext in grid: ", number(total_space, "B")]) total_shares = n * total_files add_output("Grid", T.div["Total shares in grid: ", number(total_shares)]) expansion = float(n) / float(k) total_usage = expansion * total_space add_output("Grid", T.div["Share data in grid: ", number(total_usage, "B")]) if n > num_servers: # silly configuration, causes Tahoe2 to wrap and put multiple # shares on some servers. add_output( "Servers", T.div["non-ideal: more shares than servers" " (n=%d, servers=%d)" % (n, num_servers)]) # every file has at least one share on every server buckets_per_server = total_files shares_per_server = total_files * ((1.0 * n) / num_servers) else: # if nobody is full, then no lease requests will be turned # down for lack of space, and no two shares for the same file # will share a server. Therefore the chance that any given # file has a share on any given server is n/num_servers. buckets_per_server = total_files * ((1.0 * n) / num_servers) # since each such represented file only puts one share on a # server, the total number of shares per server is the same. shares_per_server = buckets_per_server add_output( "Servers", T.div["Buckets per server: ", number(buckets_per_server)]) add_output("Servers", T.div["Shares per server: ", number(shares_per_server)]) # how much space is used on the storage servers for the shares? # the share data itself share_data_per_server = total_usage / num_servers add_output( "Servers", T.div["Share data per server: ", number(share_data_per_server, "B")]) # this is determined empirically. H=hashsize=32, for a one-segment # file and 3-of-10 encoding share_validation_per_server = 266 * shares_per_server # this could be 423*buckets_per_server, if we moved the URI # extension into a separate file, but that would actually consume # *more* space (minimum filesize is 4KiB), unless we moved all # shares for a given bucket into a single file. share_uri_extension_per_server = 423 * shares_per_server # ownership mode adds per-bucket data H = 32 # depends upon the desired security of delete/refresh caps # bucket_lease_size is the amount of data needed to keep track of # the delete/refresh caps for each bucket. bucket_lease_size = 0 client_bucket_refresh_rate = 0 owner_table_size = 0 if ownership_mode in ("B", "C", "D", "E"): bucket_lease_size = sharing_ratio * 1.0 * H if ownership_mode in ("B", "C"): # refreshes per second per client client_bucket_refresh_rate = (1.0 * n * files_per_user / seconds_per_lease) add_output( "Users", T.div["Client share refresh rate (outbound): ", number(client_bucket_refresh_rate, "Hz")]) server_bucket_refresh_rate = (client_bucket_refresh_rate * num_users / num_servers) add_output( "Servers", T.div["Server share refresh rate (inbound): ", number(server_bucket_refresh_rate, "Hz")]) if ownership_mode in ("D", "E"): # each server must maintain a bidirectional mapping from # buckets to owners. One way to implement this would be to # put a list of four-byte owner numbers into each bucket, and # a list of four-byte share numbers into each owner (although # of course we'd really just throw it into a database and let # the experts take care of the details). owner_table_size = 2 * (buckets_per_server * sharing_ratio * 4) if ownership_mode in ("E", ): # in this mode, clients must refresh one timer per server client_account_refresh_rate = (1.0 * num_servers / seconds_per_lease) add_output( "Users", T.div["Client account refresh rate (outbound): ", number(client_account_refresh_rate, "Hz")]) server_account_refresh_rate = (client_account_refresh_rate * num_users / num_servers) add_output( "Servers", T.div["Server account refresh rate (inbound): ", number(server_account_refresh_rate, "Hz")]) # TODO: buckets vs shares here is a bit wonky, but in # non-wrapping grids it shouldn't matter share_lease_per_server = bucket_lease_size * buckets_per_server share_ownertable_per_server = owner_table_size share_space_per_server = (share_data_per_server + share_validation_per_server + share_uri_extension_per_server + share_lease_per_server + share_ownertable_per_server) add_output( "Servers", T.div["Share space per server: ", number(share_space_per_server, "B"), " (data ", number(share_data_per_server, "B"), ", validation ", number(share_validation_per_server, "B"), ", UEB ", number(share_uri_extension_per_server, "B"), ", lease ", number(share_lease_per_server, "B"), ", ownertable ", number(share_ownertable_per_server, "B"), ")", ]) # rates client_download_share_rate = download_rate * k client_download_byte_rate = download_rate * file_size add_output( "Users", T.div["download rate: shares = ", number(client_download_share_rate, "Hz"), " , bytes = ", number(client_download_byte_rate, "Bps"), ]) total_file_check_rate = 1.0 * total_files / file_check_interval client_check_share_rate = total_file_check_rate / num_users add_output( "Users", T.div["file check rate: shares = ", number(client_check_share_rate, "Hz"), " (interval = %s)" % number(1 / client_check_share_rate, "s"), ]) client_upload_share_rate = upload_rate * n # TODO: doesn't include overhead client_upload_byte_rate = upload_rate * file_size * expansion add_output( "Users", T.div["upload rate: shares = ", number(client_upload_share_rate, "Hz"), " , bytes = ", number(client_upload_byte_rate, "Bps"), ]) client_delete_share_rate = delete_rate * n server_inbound_share_rate = (client_upload_share_rate * num_users / num_servers) server_inbound_byte_rate = (client_upload_byte_rate * num_users / num_servers) add_output( "Servers", T.div["upload rate (inbound): shares = ", number(server_inbound_share_rate, "Hz"), " , bytes = ", number(server_inbound_byte_rate, "Bps"), ]) add_output( "Servers", T.div["share check rate (inbound): ", number(total_file_check_rate * n / num_servers, "Hz"), ]) server_share_modify_rate = ( (client_upload_share_rate + client_delete_share_rate) * num_users / num_servers) add_output( "Servers", T.div["share modify rate: shares = ", number(server_share_modify_rate, "Hz"), ]) server_outbound_share_rate = (client_download_share_rate * num_users / num_servers) server_outbound_byte_rate = (client_download_byte_rate * num_users / num_servers) add_output( "Servers", T.div["download rate (outbound): shares = ", number(server_outbound_share_rate, "Hz"), " , bytes = ", number(server_outbound_byte_rate, "Bps"), ]) total_share_space = num_servers * share_space_per_server add_output( "Grid", T.div["Share space consumed: ", number(total_share_space, "B")]) add_output( "Grid", T.div[" %% validation: %.2f%%" % (100.0 * share_validation_per_server / share_space_per_server)]) add_output( "Grid", T.div[" %% uri-extension: %.2f%%" % (100.0 * share_uri_extension_per_server / share_space_per_server)]) add_output( "Grid", T.div[" %% lease data: %.2f%%" % (100.0 * share_lease_per_server / share_space_per_server)]) add_output( "Grid", T.div[" %% owner data: %.2f%%" % (100.0 * share_ownertable_per_server / share_space_per_server)]) add_output( "Grid", T.div[" %% share data: %.2f%%" % (100.0 * share_data_per_server / share_space_per_server)]) add_output( "Grid", T.div["file check rate: ", number(total_file_check_rate, "Hz")]) total_drives = max( mathutil.div_ceil(int(total_share_space), int(drive_size)), num_servers) add_output( "Drives", T.div["Total drives: ", number(total_drives), " drives"]) drives_per_server = mathutil.div_ceil(total_drives, num_servers) add_output("Servers", T.div["Drives per server: ", drives_per_server]) # costs if drive_size == 750 * 1e9: add_output("Servers", T.div["750GB drive: $250 each"]) drive_cost = 250 else: add_output("Servers", T.div[T.b["unknown cost per drive, assuming $100"]]) drive_cost = 100 if drives_per_server <= 4: add_output("Servers", T.div["1U box with <= 4 drives: $1500"]) server_cost = 1500 # typical 1U box elif drives_per_server <= 12: add_output("Servers", T.div["2U box with <= 12 drives: $2500"]) server_cost = 2500 # 2U box else: add_output( "Servers", T.div[T.b["Note: too many drives per server, " "assuming $3000"]]) server_cost = 3000 server_capital_cost = (server_cost + drives_per_server * drive_cost) total_server_cost = float(num_servers * server_capital_cost) add_output( "Servers", T.div["Capital cost per server: $", server_capital_cost]) add_output( "Grid", T.div["Capital cost for all servers: $", number(total_server_cost)]) # $70/Mbps/mo # $44/server/mo power+space server_bandwidth = max(server_inbound_byte_rate, server_outbound_byte_rate) server_bandwidth_mbps = mathutil.div_ceil( int(server_bandwidth * 8), int(1e6)) server_monthly_cost = 70 * server_bandwidth_mbps + 44 add_output( "Servers", T.div["Monthly cost per server: $", server_monthly_cost]) add_output( "Users", T.div["Capital cost per user: $", number(total_server_cost / num_users)]) # reliability any_drive_failure_rate = total_drives * drive_failure_rate any_drive_MTBF = 1 // any_drive_failure_rate # in seconds any_drive_MTBF_days = any_drive_MTBF / 86400 add_output( "Drives", T.div["MTBF (any drive): ", number(any_drive_MTBF_days), " days"]) drive_replacement_monthly_cost = (float(drive_cost) * any_drive_failure_rate * 30 * 86400) add_output( "Grid", T.div["Monthly cost of replacing drives: $", number(drive_replacement_monthly_cost)]) total_server_monthly_cost = float(num_servers * server_monthly_cost + drive_replacement_monthly_cost) add_output( "Grid", T.div["Monthly cost for all servers: $", number(total_server_monthly_cost)]) add_output( "Users", T.div["Monthly cost per user: $", number(total_server_monthly_cost / num_users)]) # availability file_dBA = self.file_availability(k, n, server_dBA) user_files_dBA = self.many_files_availability( file_dBA, files_per_user) all_files_dBA = self.many_files_availability(file_dBA, total_files) add_output( "Users", T.div["availability of: ", "arbitrary file = %d dBA, " % file_dBA, "all files of user1 = %d dBA, " % user_files_dBA, "all files in grid = %d dBA" % all_files_dBA, ], ) time_until_files_lost = (n - k + 1) / any_drive_failure_rate add_output( "Grid", T.div["avg time until files are lost: ", number(time_until_files_lost, "s"), ", ", number(time_until_files_lost / 86400, " days"), ]) share_data_loss_rate = any_drive_failure_rate * drive_size add_output( "Grid", T.div["share data loss rate: ", number(share_data_loss_rate, "Bps")]) # the worst-case survival numbers occur when we do a file check # and the file is just above the threshold for repair (so we # decide to not repair it). The question is then: what is the # chance that the file will decay so badly before the next check # that we can't recover it? The resulting probability is per # check interval. # Note that the chances of us getting into this situation are low. P_disk_failure_during_interval = (drive_failure_rate * file_check_interval) disk_failure_dBF = 10 * math.log10(P_disk_failure_during_interval) disk_failure_dBA = -disk_failure_dBF file_survives_dBA = self.file_availability(k, repair_threshold, disk_failure_dBA) user_files_survives_dBA = self.many_files_availability( \ file_survives_dBA, files_per_user) all_files_survives_dBA = self.many_files_availability( \ file_survives_dBA, total_files) add_output( "Users", T.div["survival of: ", "arbitrary file = %d dBA, " % file_survives_dBA, "all files of user1 = %d dBA, " % user_files_survives_dBA, "all files in grid = %d dBA" % all_files_survives_dBA, " (per worst-case check interval)", ]) all_sections = [] all_sections.append(build_section("Users")) all_sections.append(build_section("Servers")) all_sections.append(build_section("Drives")) if "Grid" in sections: all_sections.append(build_section("Grid")) f = T.form(action=".", method="post", enctype="multipart/form-data") if filled: action = "Recompute" else: action = "Compute" f = f[T.input(type="hidden", name="filled", value="true"), T.input(type="submit", value=action), all_sections, ] try: from allmydata import reliability # we import this just to test to see if the page is available _hush_pyflakes = reliability del _hush_pyflakes f = [T.div[T.a(href="../reliability")["Reliability Math"]], f] except ImportError: pass return f
def do_forms(self, getarg): filled = getarg("filled", bool) def get_and_set(name, options, default=None, astype=int): current_value = getarg(name, astype) i_select = T.select(name=name) for (count, description) in options: count = astype(count) if ((current_value is not None and count == current_value) or (current_value is None and count == default)): o = T.option(value=str(count), selected="true")[description] else: o = T.option(value=str(count))[description] i_select = i_select[o] if current_value is None: current_value = default return current_value, i_select sections = {} def add_input(section, text, entry): if section not in sections: sections[section] = [] sections[section].extend([T.div[text, ": ", entry], "\n"]) def add_output(section, entry): if section not in sections: sections[section] = [] sections[section].extend([entry, "\n"]) def build_section(section): return T.fieldset[T.legend[section], sections[section]] def number(value, suffix=""): scaling = 1 if value < 1: fmt = "%1.2g%s" elif value < 100: fmt = "%.1f%s" elif value < 1000: fmt = "%d%s" elif value < 1e6: fmt = "%.2fk%s"; scaling = 1e3 elif value < 1e9: fmt = "%.2fM%s"; scaling = 1e6 elif value < 1e12: fmt = "%.2fG%s"; scaling = 1e9 elif value < 1e15: fmt = "%.2fT%s"; scaling = 1e12 elif value < 1e18: fmt = "%.2fP%s"; scaling = 1e15 else: fmt = "huge! %g%s" return fmt % (value / scaling, suffix) user_counts = [(5, "5 users"), (50, "50 users"), (200, "200 users"), (1000, "1k users"), (10000, "10k users"), (50000, "50k users"), (100000, "100k users"), (500000, "500k users"), (1000000, "1M users"), ] num_users, i_num_users = get_and_set("num_users", user_counts, 50000) add_input("Users", "How many users are on this network?", i_num_users) files_per_user_counts = [(100, "100 files"), (1000, "1k files"), (10000, "10k files"), (100000, "100k files"), (1e6, "1M files"), ] files_per_user, i_files_per_user = get_and_set("files_per_user", files_per_user_counts, 1000) add_input("Users", "How many files for each user? (avg)", i_files_per_user) space_per_user_sizes = [(1e6, "1MB"), (10e6, "10MB"), (100e6, "100MB"), (200e6, "200MB"), (1e9, "1GB"), (2e9, "2GB"), (5e9, "5GB"), (10e9, "10GB"), (100e9, "100GB"), (1e12, "1TB"), ] # current allmydata average utilization 127MB per user space_per_user, i_space_per_user = get_and_set("space_per_user", space_per_user_sizes, 200e6) add_input("Users", "How much data for each user? (avg)", i_space_per_user) sharing_ratios = [(1.0, "1.0x"), (1.1, "1.1x"), (2.0, "2.0x"), ] sharing_ratio, i_sharing_ratio = get_and_set("sharing_ratio", sharing_ratios, 1.0, float) add_input("Users", "What is the sharing ratio? (1.0x is no-sharing and" " no convergence)", i_sharing_ratio) # Encoding parameters encoding_choices = [("3-of-10-5", "3.3x (3-of-10, repair below 5)"), ("3-of-10-8", "3.3x (3-of-10, repair below 8)"), ("5-of-10-7", "2x (5-of-10, repair below 7)"), ("8-of-10-9", "1.25x (8-of-10, repair below 9)"), ("27-of-30-28", "1.1x (27-of-30, repair below 28"), ("25-of-100-50", "4x (25-of-100, repair below 50)"), ] encoding_parameters, i_encoding_parameters = \ get_and_set("encoding_parameters", encoding_choices, "3-of-10-5", str) encoding_pieces = encoding_parameters.split("-") k = int(encoding_pieces[0]) assert encoding_pieces[1] == "of" n = int(encoding_pieces[2]) # we repair the file when the number of available shares drops below # this value repair_threshold = int(encoding_pieces[3]) add_input("Servers", "What are the default encoding parameters?", i_encoding_parameters) # Server info num_server_choices = [ (5, "5 servers"), (10, "10 servers"), (15, "15 servers"), (30, "30 servers"), (50, "50 servers"), (100, "100 servers"), (200, "200 servers"), (300, "300 servers"), (500, "500 servers"), (1000, "1k servers"), (2000, "2k servers"), (5000, "5k servers"), (10e3, "10k servers"), (100e3, "100k servers"), (1e6, "1M servers"), ] num_servers, i_num_servers = \ get_and_set("num_servers", num_server_choices, 30, int) add_input("Servers", "How many servers are there?", i_num_servers) # availability is measured in dBA = -dBF, where 0dBF is 100% failure, # 10dBF is 10% failure, 20dBF is 1% failure, etc server_dBA_choices = [ (10, "90% [10dBA] (2.4hr/day)"), (13, "95% [13dBA] (1.2hr/day)"), (20, "99% [20dBA] (14min/day or 3.5days/year)"), (23, "99.5% [23dBA] (7min/day or 1.75days/year)"), (30, "99.9% [30dBA] (87sec/day or 9hours/year)"), (40, "99.99% [40dBA] (60sec/week or 53min/year)"), (50, "99.999% [50dBA] (5min per year)"), ] server_dBA, i_server_availability = \ get_and_set("server_availability", server_dBA_choices, 20, int) add_input("Servers", "What is the server availability?", i_server_availability) drive_MTBF_choices = [ (40, "40,000 Hours"), ] drive_MTBF, i_drive_MTBF = \ get_and_set("drive_MTBF", drive_MTBF_choices, 40, int) add_input("Drives", "What is the hard drive MTBF?", i_drive_MTBF) # http://www.tgdaily.com/content/view/30990/113/ # http://labs.google.com/papers/disk_failures.pdf # google sees: # 1.7% of the drives they replaced were 0-1 years old # 8% of the drives they repalced were 1-2 years old # 8.6% were 2-3 years old # 6% were 3-4 years old, about 8% were 4-5 years old drive_size_choices = [ (100, "100 GB"), (250, "250 GB"), (500, "500 GB"), (750, "750 GB"), ] drive_size, i_drive_size = \ get_and_set("drive_size", drive_size_choices, 750, int) drive_size = drive_size * 1e9 add_input("Drives", "What is the capacity of each hard drive?", i_drive_size) drive_failure_model_choices = [ ("E", "Exponential"), ("U", "Uniform"), ] drive_failure_model, i_drive_failure_model = \ get_and_set("drive_failure_model", drive_failure_model_choices, "E", str) add_input("Drives", "How should we model drive failures?", i_drive_failure_model) # drive_failure_rate is in failures per second if drive_failure_model == "E": drive_failure_rate = 1.0 / (drive_MTBF * 1000 * 3600) else: drive_failure_rate = 0.5 / (drive_MTBF * 1000 * 3600) # deletion/gc/ownership mode ownership_choices = [ ("A", "no deletion, no gc, no owners"), ("B", "deletion, no gc, no owners"), ("C", "deletion, share timers, no owners"), ("D", "deletion, no gc, yes owners"), ("E", "deletion, owner timers"), ] ownership_mode, i_ownership_mode = \ get_and_set("ownership_mode", ownership_choices, "A", str) add_input("Servers", "What is the ownership mode?", i_ownership_mode) # client access behavior access_rates = [ (1, "one file per day"), (10, "10 files per day"), (100, "100 files per day"), (1000, "1k files per day"), (10e3, "10k files per day"), (100e3, "100k files per day"), ] download_files_per_day, i_download_rate = \ get_and_set("download_rate", access_rates, 100, int) add_input("Users", "How many files are downloaded per day?", i_download_rate) download_rate = 1.0 * download_files_per_day / (24*60*60) upload_files_per_day, i_upload_rate = \ get_and_set("upload_rate", access_rates, 10, int) add_input("Users", "How many files are uploaded per day?", i_upload_rate) upload_rate = 1.0 * upload_files_per_day / (24*60*60) delete_files_per_day, i_delete_rate = \ get_and_set("delete_rate", access_rates, 10, int) add_input("Users", "How many files are deleted per day?", i_delete_rate) delete_rate = 1.0 * delete_files_per_day / (24*60*60) # the value is in days lease_timers = [ (1, "one refresh per day"), (7, "one refresh per week"), ] lease_timer, i_lease = \ get_and_set("lease_timer", lease_timers, 7, int) add_input("Users", "How frequently do clients refresh files or accounts? " "(if necessary)", i_lease) seconds_per_lease = 24*60*60*lease_timer check_timer_choices = [ (1, "every week"), (4, "every month"), (8, "every two months"), (16, "every four months"), ] check_timer, i_check_timer = \ get_and_set("check_timer", check_timer_choices, 4, int) add_input("Users", "How frequently should we check on each file?", i_check_timer) file_check_interval = check_timer * 7 * 24 * 3600 if filled: add_output("Users", T.div["Total users: %s" % number(num_users)]) add_output("Users", T.div["Files per user: %s" % number(files_per_user)]) file_size = 1.0 * space_per_user / files_per_user add_output("Users", T.div["Average file size: ", number(file_size)]) total_files = num_users * files_per_user / sharing_ratio add_output("Grid", T.div["Total number of files in grid: ", number(total_files)]) total_space = num_users * space_per_user / sharing_ratio add_output("Grid", T.div["Total volume of plaintext in grid: ", number(total_space, "B")]) total_shares = n * total_files add_output("Grid", T.div["Total shares in grid: ", number(total_shares)]) expansion = float(n) / float(k) total_usage = expansion * total_space add_output("Grid", T.div["Share data in grid: ", number(total_usage, "B")]) if n > num_servers: # silly configuration, causes Tahoe2 to wrap and put multiple # shares on some servers. add_output("Servers", T.div["non-ideal: more shares than servers" " (n=%d, servers=%d)" % (n, num_servers)]) # every file has at least one share on every server buckets_per_server = total_files shares_per_server = total_files * ((1.0 * n) / num_servers) else: # if nobody is full, then no lease requests will be turned # down for lack of space, and no two shares for the same file # will share a server. Therefore the chance that any given # file has a share on any given server is n/num_servers. buckets_per_server = total_files * ((1.0 * n) / num_servers) # since each such represented file only puts one share on a # server, the total number of shares per server is the same. shares_per_server = buckets_per_server add_output("Servers", T.div["Buckets per server: ", number(buckets_per_server)]) add_output("Servers", T.div["Shares per server: ", number(shares_per_server)]) # how much space is used on the storage servers for the shares? # the share data itself share_data_per_server = total_usage / num_servers add_output("Servers", T.div["Share data per server: ", number(share_data_per_server, "B")]) # this is determined empirically. H=hashsize=32, for a one-segment # file and 3-of-10 encoding share_validation_per_server = 266 * shares_per_server # this could be 423*buckets_per_server, if we moved the URI # extension into a separate file, but that would actually consume # *more* space (minimum filesize is 4KiB), unless we moved all # shares for a given bucket into a single file. share_uri_extension_per_server = 423 * shares_per_server # ownership mode adds per-bucket data H = 32 # depends upon the desired security of delete/refresh caps # bucket_lease_size is the amount of data needed to keep track of # the delete/refresh caps for each bucket. bucket_lease_size = 0 client_bucket_refresh_rate = 0 owner_table_size = 0 if ownership_mode in ("B", "C", "D", "E"): bucket_lease_size = sharing_ratio * 1.0 * H if ownership_mode in ("B", "C"): # refreshes per second per client client_bucket_refresh_rate = (1.0 * n * files_per_user / seconds_per_lease) add_output("Users", T.div["Client share refresh rate (outbound): ", number(client_bucket_refresh_rate, "Hz")]) server_bucket_refresh_rate = (client_bucket_refresh_rate * num_users / num_servers) add_output("Servers", T.div["Server share refresh rate (inbound): ", number(server_bucket_refresh_rate, "Hz")]) if ownership_mode in ("D", "E"): # each server must maintain a bidirectional mapping from # buckets to owners. One way to implement this would be to # put a list of four-byte owner numbers into each bucket, and # a list of four-byte share numbers into each owner (although # of course we'd really just throw it into a database and let # the experts take care of the details). owner_table_size = 2*(buckets_per_server * sharing_ratio * 4) if ownership_mode in ("E",): # in this mode, clients must refresh one timer per server client_account_refresh_rate = (1.0 * num_servers / seconds_per_lease) add_output("Users", T.div["Client account refresh rate (outbound): ", number(client_account_refresh_rate, "Hz")]) server_account_refresh_rate = (client_account_refresh_rate * num_users / num_servers) add_output("Servers", T.div["Server account refresh rate (inbound): ", number(server_account_refresh_rate, "Hz")]) # TODO: buckets vs shares here is a bit wonky, but in # non-wrapping grids it shouldn't matter share_lease_per_server = bucket_lease_size * buckets_per_server share_ownertable_per_server = owner_table_size share_space_per_server = (share_data_per_server + share_validation_per_server + share_uri_extension_per_server + share_lease_per_server + share_ownertable_per_server) add_output("Servers", T.div["Share space per server: ", number(share_space_per_server, "B"), " (data ", number(share_data_per_server, "B"), ", validation ", number(share_validation_per_server, "B"), ", UEB ", number(share_uri_extension_per_server, "B"), ", lease ", number(share_lease_per_server, "B"), ", ownertable ", number(share_ownertable_per_server, "B"), ")", ]) # rates client_download_share_rate = download_rate * k client_download_byte_rate = download_rate * file_size add_output("Users", T.div["download rate: shares = ", number(client_download_share_rate, "Hz"), " , bytes = ", number(client_download_byte_rate, "Bps"), ]) total_file_check_rate = 1.0 * total_files / file_check_interval client_check_share_rate = total_file_check_rate / num_users add_output("Users", T.div["file check rate: shares = ", number(client_check_share_rate, "Hz"), " (interval = %s)" % number(1 / client_check_share_rate, "s"), ]) client_upload_share_rate = upload_rate * n # TODO: doesn't include overhead client_upload_byte_rate = upload_rate * file_size * expansion add_output("Users", T.div["upload rate: shares = ", number(client_upload_share_rate, "Hz"), " , bytes = ", number(client_upload_byte_rate, "Bps"), ]) client_delete_share_rate = delete_rate * n server_inbound_share_rate = (client_upload_share_rate * num_users / num_servers) server_inbound_byte_rate = (client_upload_byte_rate * num_users / num_servers) add_output("Servers", T.div["upload rate (inbound): shares = ", number(server_inbound_share_rate, "Hz"), " , bytes = ", number(server_inbound_byte_rate, "Bps"), ]) add_output("Servers", T.div["share check rate (inbound): ", number(total_file_check_rate * n / num_servers, "Hz"), ]) server_share_modify_rate = ((client_upload_share_rate + client_delete_share_rate) * num_users / num_servers) add_output("Servers", T.div["share modify rate: shares = ", number(server_share_modify_rate, "Hz"), ]) server_outbound_share_rate = (client_download_share_rate * num_users / num_servers) server_outbound_byte_rate = (client_download_byte_rate * num_users / num_servers) add_output("Servers", T.div["download rate (outbound): shares = ", number(server_outbound_share_rate, "Hz"), " , bytes = ", number(server_outbound_byte_rate, "Bps"), ]) total_share_space = num_servers * share_space_per_server add_output("Grid", T.div["Share space consumed: ", number(total_share_space, "B")]) add_output("Grid", T.div[" %% validation: %.2f%%" % (100.0 * share_validation_per_server / share_space_per_server)]) add_output("Grid", T.div[" %% uri-extension: %.2f%%" % (100.0 * share_uri_extension_per_server / share_space_per_server)]) add_output("Grid", T.div[" %% lease data: %.2f%%" % (100.0 * share_lease_per_server / share_space_per_server)]) add_output("Grid", T.div[" %% owner data: %.2f%%" % (100.0 * share_ownertable_per_server / share_space_per_server)]) add_output("Grid", T.div[" %% share data: %.2f%%" % (100.0 * share_data_per_server / share_space_per_server)]) add_output("Grid", T.div["file check rate: ", number(total_file_check_rate, "Hz")]) total_drives = max(mathutil.div_ceil(int(total_share_space), int(drive_size)), num_servers) add_output("Drives", T.div["Total drives: ", number(total_drives), " drives"]) drives_per_server = mathutil.div_ceil(total_drives, num_servers) add_output("Servers", T.div["Drives per server: ", drives_per_server]) # costs if drive_size == 750 * 1e9: add_output("Servers", T.div["750GB drive: $250 each"]) drive_cost = 250 else: add_output("Servers", T.div[T.b["unknown cost per drive, assuming $100"]]) drive_cost = 100 if drives_per_server <= 4: add_output("Servers", T.div["1U box with <= 4 drives: $1500"]) server_cost = 1500 # typical 1U box elif drives_per_server <= 12: add_output("Servers", T.div["2U box with <= 12 drives: $2500"]) server_cost = 2500 # 2U box else: add_output("Servers", T.div[T.b["Note: too many drives per server, " "assuming $3000"]]) server_cost = 3000 server_capital_cost = (server_cost + drives_per_server * drive_cost) total_server_cost = float(num_servers * server_capital_cost) add_output("Servers", T.div["Capital cost per server: $", server_capital_cost]) add_output("Grid", T.div["Capital cost for all servers: $", number(total_server_cost)]) # $70/Mbps/mo # $44/server/mo power+space server_bandwidth = max(server_inbound_byte_rate, server_outbound_byte_rate) server_bandwidth_mbps = mathutil.div_ceil(int(server_bandwidth*8), int(1e6)) server_monthly_cost = 70*server_bandwidth_mbps + 44 add_output("Servers", T.div["Monthly cost per server: $", server_monthly_cost]) add_output("Users", T.div["Capital cost per user: $", number(total_server_cost / num_users)]) # reliability any_drive_failure_rate = total_drives * drive_failure_rate any_drive_MTBF = 1 // any_drive_failure_rate # in seconds any_drive_MTBF_days = any_drive_MTBF / 86400 add_output("Drives", T.div["MTBF (any drive): ", number(any_drive_MTBF_days), " days"]) drive_replacement_monthly_cost = (float(drive_cost) * any_drive_failure_rate *30*86400) add_output("Grid", T.div["Monthly cost of replacing drives: $", number(drive_replacement_monthly_cost)]) total_server_monthly_cost = float(num_servers * server_monthly_cost + drive_replacement_monthly_cost) add_output("Grid", T.div["Monthly cost for all servers: $", number(total_server_monthly_cost)]) add_output("Users", T.div["Monthly cost per user: $", number(total_server_monthly_cost / num_users)]) # availability file_dBA = self.file_availability(k, n, server_dBA) user_files_dBA = self.many_files_availability(file_dBA, files_per_user) all_files_dBA = self.many_files_availability(file_dBA, total_files) add_output("Users", T.div["availability of: ", "arbitrary file = %d dBA, " % file_dBA, "all files of user1 = %d dBA, " % user_files_dBA, "all files in grid = %d dBA" % all_files_dBA, ], ) time_until_files_lost = (n-k+1) / any_drive_failure_rate add_output("Grid", T.div["avg time until files are lost: ", number(time_until_files_lost, "s"), ", ", number(time_until_files_lost/86400, " days"), ]) share_data_loss_rate = any_drive_failure_rate * drive_size add_output("Grid", T.div["share data loss rate: ", number(share_data_loss_rate,"Bps")]) # the worst-case survival numbers occur when we do a file check # and the file is just above the threshold for repair (so we # decide to not repair it). The question is then: what is the # chance that the file will decay so badly before the next check # that we can't recover it? The resulting probability is per # check interval. # Note that the chances of us getting into this situation are low. P_disk_failure_during_interval = (drive_failure_rate * file_check_interval) disk_failure_dBF = 10*math.log10(P_disk_failure_during_interval) disk_failure_dBA = -disk_failure_dBF file_survives_dBA = self.file_availability(k, repair_threshold, disk_failure_dBA) user_files_survives_dBA = self.many_files_availability( \ file_survives_dBA, files_per_user) all_files_survives_dBA = self.many_files_availability( \ file_survives_dBA, total_files) add_output("Users", T.div["survival of: ", "arbitrary file = %d dBA, " % file_survives_dBA, "all files of user1 = %d dBA, " % user_files_survives_dBA, "all files in grid = %d dBA" % all_files_survives_dBA, " (per worst-case check interval)", ]) all_sections = [] all_sections.append(build_section("Users")) all_sections.append(build_section("Servers")) all_sections.append(build_section("Drives")) if "Grid" in sections: all_sections.append(build_section("Grid")) f = T.form(action=".", method="post", enctype="multipart/form-data") if filled: action = "Recompute" else: action = "Compute" f = f[T.input(type="hidden", name="filled", value="true"), T.input(type="submit", value=action), all_sections, ] try: from allmydata import reliability # we import this just to test to see if the page is available _hush_pyflakes = reliability del _hush_pyflakes f = [T.div[T.a(href="../reliability")["Reliability Math"]], f] except ImportError: pass return f
def _setup_encoding_parameters(self): """ I set up the encoding parameters, including k, n, the number of segments associated with this file, and the segment decoders. """ (seqnum, root_hash, IV, segsize, datalength, k, n, known_prefix, offsets_tuple) = self.verinfo self._required_shares = k self._total_shares = n self._segment_size = segsize self._data_length = datalength if not IV: self._version = MDMF_VERSION else: self._version = SDMF_VERSION if datalength and segsize: self._num_segments = mathutil.div_ceil(datalength, segsize) self._tail_data_size = datalength % segsize else: self._num_segments = 0 self._tail_data_size = 0 self._segment_decoder = codec.CRSDecoder() self._segment_decoder.set_params(segsize, k, n) if not self._tail_data_size: self._tail_data_size = segsize self._tail_segment_size = mathutil.next_multiple( self._tail_data_size, self._required_shares) if self._tail_segment_size == self._segment_size: self._tail_decoder = self._segment_decoder else: self._tail_decoder = codec.CRSDecoder() self._tail_decoder.set_params(self._tail_segment_size, self._required_shares, self._total_shares) self.log("got encoding parameters: " "k: %d " "n: %d " "%d segments of %d bytes each (%d byte tail segment)" % \ (k, n, self._num_segments, self._segment_size, self._tail_segment_size)) # Our last task is to tell the downloader where to start and # where to stop. We use three parameters for that: # - self._start_segment: the segment that we need to start # downloading from. # - self._current_segment: the next segment that we need to # download. # - self._last_segment: The last segment that we were asked to # download. # # We say that the download is complete when # self._current_segment > self._last_segment. We use # self._start_segment and self._last_segment to know when to # strip things off of segments, and how much to strip. if self._offset: self.log("got offset: %d" % self._offset) # our start segment is the first segment containing the # offset we were given. start = self._offset // self._segment_size assert start < self._num_segments self._start_segment = start self.log("got start segment: %d" % self._start_segment) else: self._start_segment = 0 # If self._read_length is None, then we want to read the whole # file. Otherwise, we want to read only part of the file, and # need to figure out where to stop reading. if self._read_length is not None: # our end segment is the last segment containing part of the # segment that we were asked to read. self.log("got read length %d" % self._read_length) if self._read_length != 0: end_data = self._offset + self._read_length # We don't actually need to read the byte at end_data, # but the one before it. end = (end_data - 1) // self._segment_size assert end < self._num_segments self._last_segment = end else: self._last_segment = self._start_segment self.log("got end segment: %d" % self._last_segment) else: self._last_segment = self._num_segments - 1 self._current_segment = self._start_segment
def do_test(self, size, required_shares, max_shares, fewer_shares=None): data0s = [os.urandom(mathutil.div_ceil(size, required_shares)) for i in range(required_shares)] enc = CRSEncoder() enc.set_params(size, required_shares, max_shares) params = enc.get_params() assert params == (size, required_shares, max_shares) serialized_params = enc.get_serialized_params() self.assertEqual(parse_params(serialized_params), params) log.msg("params: %s" % (params,)) d = enc.encode(data0s) def _done_encoding_all(shares_and_shareids): (shares, shareids) = shares_and_shareids self.failUnlessEqual(len(shares), max_shares) self.shares = shares self.shareids = shareids d.addCallback(_done_encoding_all) if fewer_shares is not None: # also validate that the desired_shareids= parameter works desired_shareids = random.sample(list(range(max_shares)), fewer_shares) d.addCallback(lambda res: enc.encode(data0s, desired_shareids)) def _check_fewer_shares(some_shares_and_their_shareids): (some_shares, their_shareids) = some_shares_and_their_shareids self.failUnlessEqual(tuple(their_shareids), tuple(desired_shareids)) d.addCallback(_check_fewer_shares) def _decode(shares_and_shareids): (shares, shareids) = shares_and_shareids dec = CRSDecoder() dec.set_params(*params) d1 = dec.decode(shares, shareids) return d1 def _check_data(decoded_shares): self.failUnlessEqual(len(b''.join(decoded_shares)), len(b''.join(data0s))) self.failUnlessEqual(len(decoded_shares), len(data0s)) for (i, (x, y)) in enumerate(zip(data0s, decoded_shares)): self.failUnlessEqual(x, y, "%s: %r != %r.... first share was %r" % (str(i), x, y, data0s[0],)) self.failUnless(b''.join(decoded_shares) == b''.join(data0s), "%s" % ("???",)) # 0data0sclipped = tuple(data0s) # data0sclipped[-1] = # self.failUnless(tuple(decoded_shares) == tuple(data0s)) def _decode_some(res): log.msg("_decode_some") # decode with a minimal subset of the shares some_shares = self.shares[:required_shares] some_shareids = self.shareids[:required_shares] return _decode((some_shares, some_shareids)) d.addCallback(_decode_some) d.addCallback(_check_data) def _decode_some_random(res): log.msg("_decode_some_random") # use a randomly-selected minimal subset l = random.sample(list(zip(self.shares, self.shareids)), required_shares) some_shares = [ x[0] for x in l ] some_shareids = [ x[1] for x in l ] return _decode((some_shares, some_shareids)) d.addCallback(_decode_some_random) d.addCallback(_check_data) def _decode_multiple(res): log.msg("_decode_multiple") # make sure we can re-use the decoder object shares1 = random.sample(self.shares, required_shares) sharesl1 = random.sample(list(zip(self.shares, self.shareids)), required_shares) shares1 = [ x[0] for x in sharesl1 ] shareids1 = [ x[1] for x in sharesl1 ] sharesl2 = random.sample(list(zip(self.shares, self.shareids)), required_shares) shares2 = [ x[0] for x in sharesl2 ] shareids2 = [ x[1] for x in sharesl2 ] dec = CRSDecoder() dec.set_params(*params) d1 = dec.decode(shares1, shareids1) d1.addCallback(_check_data) d1.addCallback(lambda res: dec.decode(shares2, shareids2)) d1.addCallback(_check_data) return d1 d.addCallback(_decode_multiple) return d