def _update(self, data, offset):
"""
I update the mutable file version represented by this particular
IMutableVersion by inserting the data in data at the offset
offset. I return a Deferred that fires when this has been
completed.
"""
new_size = data.get_size() + offset
old_size = self.get_size()
segment_size = self._version[3]
num_old_segments = mathutil.div_ceil(old_size,
segment_size)
num_new_segments = mathutil.div_ceil(new_size,
segment_size)
log.msg("got %d old segments, %d new segments" % \
(num_old_segments, num_new_segments))
# We do a whole file re-encode if the file is an SDMF file.
if self._version[2]: # version[2] == SDMF salt, which MDMF lacks
log.msg("doing re-encode instead of in-place update")
return self._do_modify_update(data, offset)
# Otherwise, we can replace just the parts that are changing.
log.msg("updating in place")
d = self._do_update_update(data, offset)
d.addCallback(self._decode_and_decrypt_segments, data, offset)
d.addCallback(self._build_uploadable_and_finish, data, offset)
return d
def _update(self, data, offset):
"""
I update the mutable file version represented by this particular
IMutableVersion by inserting the data in data at the offset
offset. I return a Deferred that fires when this has been
completed.
"""
new_size = data.get_size() + offset
old_size = self.get_size()
segment_size = self._version[3]
num_old_segments = mathutil.div_ceil(old_size,
segment_size)
num_new_segments = mathutil.div_ceil(new_size,
segment_size)
log.msg("got %d old segments, %d new segments" % \
(num_old_segments, num_new_segments))
# We do a whole file re-encode if the file is an SDMF file.
if self._version[2]: # version[2] == SDMF salt, which MDMF lacks
log.msg("doing re-encode instead of in-place update")
return self._do_modify_update(data, offset)
# Otherwise, we can replace just the parts that are changing.
log.msg("updating in place")
d = self._do_update_update(data, offset)
d.addCallback(self._decode_and_decrypt_segments, data, offset)
d.addCallback(self._build_uploadable_and_finish, data, offset)
return d
def _got_all_encoding_parameters(self, params):
assert not self._codec
k, happy, n, segsize = params
self.required_shares = k
self.min_happiness = happy
self.num_shares = n
self.segment_size = segsize
self.log("got encoding parameters: %d/%d/%d %d" %
(k, happy, n, segsize))
self.log("now setting up codec")
assert self.segment_size % self.required_shares == 0
self.num_segments = mathutil.div_ceil(self.file_size,
self.segment_size)
self._codec = CRSEncoder()
self._codec.set_params(self.segment_size, self.required_shares,
self.num_shares)
data = self.uri_extension_data
data['codec_name'] = self._codec.get_encoder_type()
data['codec_params'] = self._codec.get_serialized_params()
data['size'] = self.file_size
data['segment_size'] = self.segment_size
self.share_size = mathutil.div_ceil(self.file_size,
self.required_shares)
data['num_segments'] = self.num_segments
data['needed_shares'] = self.required_shares
data['total_shares'] = self.num_shares
# the "tail" is the last segment. This segment may or may not be
# shorter than all other segments. We use the "tail codec" to handle
# it. If the tail is short, we use a different codec instance. In
# addition, the tail codec must be fed data which has been padded out
# to the right size.
tail_size = self.file_size % self.segment_size
if not tail_size:
tail_size = self.segment_size
# the tail codec is responsible for encoding tail_size bytes
padded_tail_size = mathutil.next_multiple(tail_size,
self.required_shares)
self._tail_codec = CRSEncoder()
self._tail_codec.set_params(padded_tail_size, self.required_shares,
self.num_shares)
data['tail_codec_params'] = self._tail_codec.get_serialized_params()
def _calculate_sizes(self, segment_size):
# segments of ciphertext
size = self._verifycap.size
k = self._verifycap.needed_shares
# this assert matches the one in encode.py:127 inside
# Encoded._got_all_encoding_parameters, where the UEB is constructed
assert segment_size % k == 0
# the last segment is usually short. We don't store a whole segsize,
# but we do pad the segment up to a multiple of k, because the
# encoder requires that.
tail_segment_size = size % segment_size
if tail_segment_size == 0:
tail_segment_size = segment_size
padded = mathutil.next_multiple(tail_segment_size, k)
tail_segment_padded = padded
num_segments = mathutil.div_ceil(size, segment_size)
# each segment is turned into N blocks. All but the last are of size
# block_size, and the last is of size tail_block_size
block_size = segment_size // k
tail_block_size = tail_segment_padded // k
return { "tail_segment_size": tail_segment_size,
"tail_segment_padded": tail_segment_padded,
"num_segments": num_segments,
"block_size": block_size,
"tail_block_size": tail_block_size
}
def _calculate_sizes(self, segment_size):
# segments of ciphertext
size = self._verifycap.size
k = self._verifycap.needed_shares
# this assert matches the one in encode.py:127 inside
# Encoded._got_all_encoding_parameters, where the UEB is constructed
assert segment_size % k == 0
# the last segment is usually short. We don't store a whole segsize,
# but we do pad the segment up to a multiple of k, because the
# encoder requires that.
tail_segment_size = size % segment_size
if tail_segment_size == 0:
tail_segment_size = segment_size
padded = mathutil.next_multiple(tail_segment_size, k)
tail_segment_padded = padded
num_segments = mathutil.div_ceil(size, segment_size)
# each segment is turned into N blocks. All but the last are of size
# block_size, and the last is of size tail_block_size
block_size = segment_size / k
tail_block_size = tail_segment_padded / k
return { "tail_segment_size": tail_segment_size,
"tail_segment_padded": tail_segment_padded,
"num_segments": num_segments,
"block_size": block_size,
"tail_block_size": tail_block_size,
}
def _got_all_encoding_parameters(self, params):
assert not self._codec
k, happy, n, segsize = params
self.required_shares = k
self.servers_of_happiness = happy
self.num_shares = n
self.segment_size = segsize
self.log("got encoding parameters: %d/%d/%d %d" % (k,happy,n, segsize))
self.log("now setting up codec")
assert self.segment_size % self.required_shares == 0
self.num_segments = mathutil.div_ceil(self.file_size,
self.segment_size)
self._codec = CRSEncoder()
self._codec.set_params(self.segment_size,
self.required_shares, self.num_shares)
data = self.uri_extension_data
data['codec_name'] = self._codec.get_encoder_type()
data['codec_params'] = self._codec.get_serialized_params()
data['size'] = self.file_size
data['segment_size'] = self.segment_size
self.share_size = mathutil.div_ceil(self.file_size,
self.required_shares)
data['num_segments'] = self.num_segments
data['needed_shares'] = self.required_shares
data['total_shares'] = self.num_shares
# the "tail" is the last segment. This segment may or may not be
# shorter than all other segments. We use the "tail codec" to handle
# it. If the tail is short, we use a different codec instance. In
# addition, the tail codec must be fed data which has been padded out
# to the right size.
tail_size = self.file_size % self.segment_size
if not tail_size:
tail_size = self.segment_size
# the tail codec is responsible for encoding tail_size bytes
padded_tail_size = mathutil.next_multiple(tail_size,
self.required_shares)
self._tail_codec = CRSEncoder()
self._tail_codec.set_params(padded_tail_size,
self.required_shares, self.num_shares)
data['tail_codec_params'] = self._tail_codec.get_serialized_params()
def set_params(self, data_size, required_shares, max_shares):
assert required_shares <= max_shares
self.data_size = data_size
self.required_shares = required_shares
self.max_shares = max_shares
self.share_size = mathutil.div_ceil(data_size, required_shares)
self.last_share_padding = mathutil.pad_size(self.share_size, required_shares)
self.encoder = zfec.Encoder(required_shares, max_shares)
def set_params(self, data_size, required_shares, max_shares):
assert required_shares <= max_shares
self.data_size = data_size
self.required_shares = required_shares
self.max_shares = max_shares
self.share_size = mathutil.div_ceil(data_size, required_shares)
self.last_share_padding = mathutil.pad_size(self.share_size, required_shares)
self.encoder = zfec.Encoder(required_shares, max_shares)
def set_params(self, data_size, required_shares, max_shares):
self.data_size = data_size
self.required_shares = required_shares
self.max_shares = max_shares
self.chunk_size = self.required_shares
self.num_chunks = mathutil.div_ceil(self.data_size, self.chunk_size)
self.share_size = self.num_chunks
self.decoder = zfec.Decoder(self.required_shares, self.max_shares)
def set_params(self, data_size, required_shares, max_shares):
self.data_size = data_size
self.required_shares = required_shares
self.max_shares = max_shares
self.chunk_size = self.required_shares
self.num_chunks = mathutil.div_ceil(self.data_size, self.chunk_size)
self.share_size = self.num_chunks
self.decoder = zfec.Decoder(self.required_shares, self.max_shares)
def remote_measure_peer_response_time(self):
# I'd like to average together several pings, but I don't want this
# phase to take more than 10 seconds. Expect worst-case latency to be
# 300ms.
results = {}
sb = self.parent.get_storage_broker()
everyone = sb.get_connected_servers()
num_pings = int(mathutil.div_ceil(10, (len(everyone) * 0.3)))
everyone = list(everyone) * num_pings
d = self._do_one_ping(None, everyone, results)
return d
def remote_measure_peer_response_time(self):
# I'd like to average together several pings, but I don't want this
# phase to take more than 10 seconds. Expect worst-case latency to be
# 300ms.
results = {}
sb = self.parent.get_storage_broker()
everyone = sb.get_connected_servers()
num_pings = int(mathutil.div_ceil(10, (len(everyone) * 0.3)))
everyone = list(everyone) * num_pings
d = self._do_one_ping(None, everyone, results)
return d
def setup_encoding_parameters(self):
segment_size = len(self.newdata)
# this must be a multiple of self.required_shares
segment_size = mathutil.next_multiple(segment_size,
self.required_shares)
self.segment_size = segment_size
if segment_size:
self.num_segments = mathutil.div_ceil(len(self.newdata),
segment_size)
else:
self.num_segments = 0
assert self.num_segments in [0, 1,] # SDMF restrictions
def setup_encoding_parameters(self):
segment_size = len(self.newdata)
# this must be a multiple of self.required_shares
segment_size = mathutil.next_multiple(segment_size,
self.required_shares)
self.segment_size = segment_size
if segment_size:
self.num_segments = mathutil.div_ceil(len(self.newdata),
segment_size)
else:
self.num_segments = 0
assert self.num_segments in [
0,
1,
] # SDMF restrictions
def test_odd_sizes(self):
for j in range(2**6):
lib = random.randrange(1, 2**8)
numos = mathutil.div_ceil(lib, 8)
bs = insecurerandstr(numos)
# zero-out unused least-sig bits
if lib%8:
b=ord(bs[-1])
b = b >> (8 - (lib%8))
b = b << (8 - (lib%8))
bs = bs[:-1] + chr(b)
asl = base62.b2a_l(bs, lib)
assert len(asl) == base62.num_chars_that_this_many_octets_encode_to(numos) # the size of the base-62 encoding must be just right
bs2l = base62.a2b_l(asl, lib)
assert len(bs2l) == numos # the size of the result must be just right
assert bs == bs2l
def _guess_offsets(self, verifycap, guessed_segment_size):
self.guessed_segment_size = guessed_segment_size
size = verifycap.size
k = verifycap.needed_shares
N = verifycap.total_shares
r = self._node._calculate_sizes(guessed_segment_size)
# num_segments, block_size/tail_block_size
# guessed_segment_size/tail_segment_size/tail_segment_padded
share_size = mathutil.div_ceil(size, k)
# share_size is the amount of block data that will be put into each
# share, summed over all segments. It does not include hashes, the
# UEB, or other overhead.
# use the upload-side code to get this as accurate as possible
ht = IncompleteHashTree(N)
num_share_hashes = len(ht.needed_hashes(0, include_leaf=True))
wbp = make_write_bucket_proxy(None, None, share_size, r["block_size"],
r["num_segments"], num_share_hashes, 0)
self._fieldsize = wbp.fieldsize
self._fieldstruct = wbp.fieldstruct
self.guessed_offsets = wbp._offsets
def calc(filesize, params=(3, 7, 10), segsize=DEFAULT_MAX_SEGMENT_SIZE):
num_shares = params[2]
if filesize <= upload.Uploader.URI_LIT_SIZE_THRESHOLD:
urisize = len(uri.LiteralFileURI("A" * filesize).to_string())
sharesize = 0
sharespace = 0
else:
u = upload.FileUploader(None) # XXX changed
u.set_params(params)
# unfortunately, Encoder doesn't currently lend itself to answering
# this question without measuring a filesize, so we have to give it a
# fake one
data = BigFakeString(filesize)
u.set_filehandle(data)
u.set_encryption_key("a" * 16)
sharesize, blocksize = u.setup_encoder()
# how much overhead?
# 0x20 bytes of offsets
# 0x04 bytes of extension length
# 0x1ad bytes of extension (=429)
# total is 465 bytes
num_segments = mathutil.div_ceil(filesize, segsize)
num_share_hashes = int(
math.log(mathutil.next_power_of_k(num_shares, 2), 2)) + 1
sharesize = storage.allocated_size(sharesize, num_segments,
num_share_hashes, 429)
sharespace = num_shares * roundup(sharesize)
urisize = len(
uri.pack_uri(storage_index="a" * 32,
key="a" * 16,
uri_extension_hash="a" * 32,
needed_shares=params[0],
total_shares=params[2],
size=filesize))
return urisize, sharesize, sharespace
def calc(filesize, params=(3,7,10), segsize=DEFAULT_MAX_SEGMENT_SIZE):
num_shares = params[2]
if filesize <= upload.Uploader.URI_LIT_SIZE_THRESHOLD:
urisize = len(uri.LiteralFileURI("A"*filesize).to_string())
sharesize = 0
sharespace = 0
else:
u = upload.FileUploader(None) # XXX changed
u.set_params(params)
# unfortunately, Encoder doesn't currently lend itself to answering
# this question without measuring a filesize, so we have to give it a
# fake one
data = BigFakeString(filesize)
u.set_filehandle(data)
u.set_encryption_key("a"*16)
sharesize, blocksize = u.setup_encoder()
# how much overhead?
# 0x20 bytes of offsets
# 0x04 bytes of extension length
# 0x1ad bytes of extension (=429)
# total is 465 bytes
num_segments = mathutil.div_ceil(filesize, segsize)
num_share_hashes = int(math.log(mathutil.next_power_of_k(num_shares, 2),
2)) + 1
sharesize = storage.allocated_size(sharesize, num_segments,
num_share_hashes,
429)
sharespace = num_shares * roundup(sharesize)
urisize = len(uri.pack_uri(storage_index="a"*32,
key="a"*16,
uri_extension_hash="a"*32,
needed_shares=params[0],
total_shares=params[2],
size=filesize))
return urisize, sharesize, sharespace
def _setup_encoding_parameters(self):
"""
I set up the encoding parameters, including k, n, the number
of segments associated with this file, and the segment decoders.
"""
(seqnum,
root_hash,
IV,
segsize,
datalength,
k,
n,
known_prefix,
offsets_tuple) = self.verinfo
self._required_shares = k
self._total_shares = n
self._segment_size = segsize
#self._data_length = datalength # set during __init__()
if not IV:
self._version = MDMF_VERSION
else:
self._version = SDMF_VERSION
if datalength and segsize:
self._num_segments = mathutil.div_ceil(datalength, segsize)
self._tail_data_size = datalength % segsize
else:
self._num_segments = 0
self._tail_data_size = 0
self._segment_decoder = codec.CRSDecoder()
self._segment_decoder.set_params(segsize, k, n)
if not self._tail_data_size:
self._tail_data_size = segsize
self._tail_segment_size = mathutil.next_multiple(self._tail_data_size,
self._required_shares)
if self._tail_segment_size == self._segment_size:
self._tail_decoder = self._segment_decoder
else:
self._tail_decoder = codec.CRSDecoder()
self._tail_decoder.set_params(self._tail_segment_size,
self._required_shares,
self._total_shares)
self.log("got encoding parameters: "
"k: %d "
"n: %d "
"%d segments of %d bytes each (%d byte tail segment)" % \
(k, n, self._num_segments, self._segment_size,
self._tail_segment_size))
# Our last task is to tell the downloader where to start and
# where to stop. We use three parameters for that:
# - self._start_segment: the segment that we need to start
# downloading from.
# - self._current_segment: the next segment that we need to
# download.
# - self._last_segment: The last segment that we were asked to
# download.
#
# We say that the download is complete when
# self._current_segment > self._last_segment. We use
# self._start_segment and self._last_segment to know when to
# strip things off of segments, and how much to strip.
if self._offset:
self.log("got offset: %d" % self._offset)
# our start segment is the first segment containing the
# offset we were given.
start = self._offset // self._segment_size
_assert(start <= self._num_segments,
start=start, num_segments=self._num_segments,
offset=self._offset, segment_size=self._segment_size)
self._start_segment = start
self.log("got start segment: %d" % self._start_segment)
else:
self._start_segment = 0
# We might want to read only part of the file, and need to figure out
# where to stop reading. Our end segment is the last segment
# containing part of the segment that we were asked to read.
_assert(self._read_length > 0, self._read_length)
end_data = self._offset + self._read_length
# We don't actually need to read the byte at end_data, but the one
# before it.
end = (end_data - 1) // self._segment_size
_assert(0 <= end < self._num_segments,
end=end, num_segments=self._num_segments,
end_data=end_data, offset=self._offset,
read_length=self._read_length, segment_size=self._segment_size)
self._last_segment = end
self.log("got end segment: %d" % self._last_segment)
self._current_segment = self._start_segment
def roundup(size, blocksize=4096):
return blocksize * mathutil.div_ceil(size, blocksize)
def do_test(self, size, required_shares, max_shares, fewer_shares=None):
data0s = [os.urandom(mathutil.div_ceil(size, required_shares)) for i in range(required_shares)]
enc = CRSEncoder()
enc.set_params(size, required_shares, max_shares)
params = enc.get_params()
assert params == (size, required_shares, max_shares)
log.msg("params: %s" % (params,))
d = enc.encode(data0s)
def _done_encoding_all(shares_and_shareids):
(shares, shareids) = shares_and_shareids
self.failUnlessEqual(len(shares), max_shares)
self.shares = shares
self.shareids = shareids
d.addCallback(_done_encoding_all)
if fewer_shares is not None:
# also validate that the desired_shareids= parameter works
desired_shareids = random.sample(range(max_shares), fewer_shares)
d.addCallback(lambda res: enc.encode(data0s, desired_shareids))
def _check_fewer_shares(some_shares_and_their_shareids):
(some_shares, their_shareids) = some_shares_and_their_shareids
self.failUnlessEqual(tuple(their_shareids), tuple(desired_shareids))
d.addCallback(_check_fewer_shares)
def _decode(shares_and_shareids):
(shares, shareids) = shares_and_shareids
dec = CRSDecoder()
dec.set_params(*params)
d1 = dec.decode(shares, shareids)
return d1
def _check_data(decoded_shares):
self.failUnlessEqual(len(''.join(decoded_shares)), len(''.join(data0s)))
self.failUnlessEqual(len(decoded_shares), len(data0s))
for (i, (x, y)) in enumerate(zip(data0s, decoded_shares)):
self.failUnlessEqual(x, y, "%s: %r != %r.... first share was %r" % (str(i), x, y, data0s[0],))
self.failUnless(''.join(decoded_shares) == ''.join(data0s), "%s" % ("???",))
# 0data0sclipped = tuple(data0s)
# data0sclipped[-1] =
# self.failUnless(tuple(decoded_shares) == tuple(data0s))
def _decode_some(res):
log.msg("_decode_some")
# decode with a minimal subset of the shares
some_shares = self.shares[:required_shares]
some_shareids = self.shareids[:required_shares]
return _decode((some_shares, some_shareids))
d.addCallback(_decode_some)
d.addCallback(_check_data)
def _decode_some_random(res):
log.msg("_decode_some_random")
# use a randomly-selected minimal subset
l = random.sample(zip(self.shares, self.shareids), required_shares)
some_shares = [ x[0] for x in l ]
some_shareids = [ x[1] for x in l ]
return _decode((some_shares, some_shareids))
d.addCallback(_decode_some_random)
d.addCallback(_check_data)
def _decode_multiple(res):
log.msg("_decode_multiple")
# make sure we can re-use the decoder object
shares1 = random.sample(self.shares, required_shares)
sharesl1 = random.sample(zip(self.shares, self.shareids), required_shares)
shares1 = [ x[0] for x in sharesl1 ]
shareids1 = [ x[1] for x in sharesl1 ]
sharesl2 = random.sample(zip(self.shares, self.shareids), required_shares)
shares2 = [ x[0] for x in sharesl2 ]
shareids2 = [ x[1] for x in sharesl2 ]
dec = CRSDecoder()
dec.set_params(*params)
d1 = dec.decode(shares1, shareids1)
d1.addCallback(_check_data)
d1.addCallback(lambda res: dec.decode(shares2, shareids2))
d1.addCallback(_check_data)
return d1
d.addCallback(_decode_multiple)
return d
def _parse_and_validate(self, data):
self.share_size = mathutil.div_ceil(self._verifycap.size,
self._verifycap.needed_shares)
d = uri.unpack_extension(data)
# There are several kinds of things that can be found in a UEB.
# First, things that we really need to learn from the UEB in order to
# do this download. Next: things which are optional but not redundant
# -- if they are present in the UEB they will get used. Next, things
# that are optional and redundant. These things are required to be
# consistent: they don't have to be in the UEB, but if they are in
# the UEB then they will be checked for consistency with the
# already-known facts, and if they are inconsistent then an exception
# will be raised. These things aren't actually used -- they are just
# tested for consistency and ignored. Finally: things which are
# deprecated -- they ought not be in the UEB at all, and if they are
# present then a warning will be logged but they are otherwise
# ignored.
# First, things that we really need to learn from the UEB:
# segment_size, crypttext_root_hash, and share_root_hash.
self.segment_size = d['segment_size']
self.block_size = mathutil.div_ceil(self.segment_size,
self._verifycap.needed_shares)
self.num_segments = mathutil.div_ceil(self._verifycap.size,
self.segment_size)
self.tail_data_size = self._verifycap.size % self.segment_size
if not self.tail_data_size:
self.tail_data_size = self.segment_size
# padding for erasure code
self.tail_segment_size = mathutil.next_multiple(self.tail_data_size,
self._verifycap.needed_shares)
# Ciphertext hash tree root is mandatory, so that there is at most
# one ciphertext that matches this read-cap or verify-cap. The
# integrity check on the shares is not sufficient to prevent the
# original encoder from creating some shares of file A and other
# shares of file B.
self.crypttext_root_hash = d['crypttext_root_hash']
self.share_root_hash = d['share_root_hash']
# Next: things that are optional and not redundant: crypttext_hash
if d.has_key('crypttext_hash'):
self.crypttext_hash = d['crypttext_hash']
if len(self.crypttext_hash) != CRYPTO_VAL_SIZE:
raise BadURIExtension('crypttext_hash is required to be hashutil.CRYPTO_VAL_SIZE bytes, not %s bytes' % (len(self.crypttext_hash),))
# Next: things that are optional, redundant, and required to be
# consistent: codec_name, codec_params, tail_codec_params,
# num_segments, size, needed_shares, total_shares
if d.has_key('codec_name'):
if d['codec_name'] != "crs":
raise UnsupportedErasureCodec(d['codec_name'])
if d.has_key('codec_params'):
ucpss, ucpns, ucpts = codec.parse_params(d['codec_params'])
if ucpss != self.segment_size:
raise BadURIExtension("inconsistent erasure code params: "
"ucpss: %s != self.segment_size: %s" %
(ucpss, self.segment_size))
if ucpns != self._verifycap.needed_shares:
raise BadURIExtension("inconsistent erasure code params: ucpns: %s != "
"self._verifycap.needed_shares: %s" %
(ucpns, self._verifycap.needed_shares))
if ucpts != self._verifycap.total_shares:
raise BadURIExtension("inconsistent erasure code params: ucpts: %s != "
"self._verifycap.total_shares: %s" %
(ucpts, self._verifycap.total_shares))
if d.has_key('tail_codec_params'):
utcpss, utcpns, utcpts = codec.parse_params(d['tail_codec_params'])
if utcpss != self.tail_segment_size:
raise BadURIExtension("inconsistent erasure code params: utcpss: %s != "
"self.tail_segment_size: %s, self._verifycap.size: %s, "
"self.segment_size: %s, self._verifycap.needed_shares: %s"
% (utcpss, self.tail_segment_size, self._verifycap.size,
self.segment_size, self._verifycap.needed_shares))
if utcpns != self._verifycap.needed_shares:
raise BadURIExtension("inconsistent erasure code params: utcpns: %s != "
"self._verifycap.needed_shares: %s" % (utcpns,
self._verifycap.needed_shares))
if utcpts != self._verifycap.total_shares:
raise BadURIExtension("inconsistent erasure code params: utcpts: %s != "
"self._verifycap.total_shares: %s" % (utcpts,
self._verifycap.total_shares))
if d.has_key('num_segments'):
if d['num_segments'] != self.num_segments:
raise BadURIExtension("inconsistent num_segments: size: %s, "
"segment_size: %s, computed_num_segments: %s, "
"ueb_num_segments: %s" % (self._verifycap.size,
self.segment_size,
self.num_segments, d['num_segments']))
if d.has_key('size'):
if d['size'] != self._verifycap.size:
raise BadURIExtension("inconsistent size: URI size: %s, UEB size: %s" %
(self._verifycap.size, d['size']))
if d.has_key('needed_shares'):
if d['needed_shares'] != self._verifycap.needed_shares:
raise BadURIExtension("inconsistent needed shares: URI needed shares: %s, UEB "
"needed shares: %s" % (self._verifycap.total_shares,
d['needed_shares']))
if d.has_key('total_shares'):
if d['total_shares'] != self._verifycap.total_shares:
raise BadURIExtension("inconsistent total shares: URI total shares: %s, UEB "
"total shares: %s" % (self._verifycap.total_shares,
d['total_shares']))
# Finally, things that are deprecated and ignored: plaintext_hash,
# plaintext_root_hash
if d.get('plaintext_hash'):
log.msg("Found plaintext_hash in UEB. This field is deprecated for security reasons "
"and is no longer used. Ignoring. %s" % (self,))
if d.get('plaintext_root_hash'):
log.msg("Found plaintext_root_hash in UEB. This field is deprecated for security "
"reasons and is no longer used. Ignoring. %s" % (self,))
return self
def _get_share_size(self):
share_size = mathutil.div_ceil(self.file_size, self.required_shares)
overhead = self._compute_overhead()
return share_size + overhead
def _get_share_size(self):
share_size = mathutil.div_ceil(self.file_size, self.required_shares)
overhead = self._compute_overhead()
return share_size + overhead
def _parse_and_validate(self, data):
self.share_size = mathutil.div_ceil(self._verifycap.size,
self._verifycap.needed_shares)
d = uri.unpack_extension(data)
# There are several kinds of things that can be found in a UEB.
# First, things that we really need to learn from the UEB in order to
# do this download. Next: things which are optional but not redundant
# -- if they are present in the UEB they will get used. Next, things
# that are optional and redundant. These things are required to be
# consistent: they don't have to be in the UEB, but if they are in
# the UEB then they will be checked for consistency with the
# already-known facts, and if they are inconsistent then an exception
# will be raised. These things aren't actually used -- they are just
# tested for consistency and ignored. Finally: things which are
# deprecated -- they ought not be in the UEB at all, and if they are
# present then a warning will be logged but they are otherwise
# ignored.
# First, things that we really need to learn from the UEB:
# segment_size, crypttext_root_hash, and share_root_hash.
self.segment_size = d['segment_size']
self.block_size = mathutil.div_ceil(self.segment_size,
self._verifycap.needed_shares)
self.num_segments = mathutil.div_ceil(self._verifycap.size,
self.segment_size)
self.tail_data_size = self._verifycap.size % self.segment_size
if not self.tail_data_size:
self.tail_data_size = self.segment_size
# padding for erasure code
self.tail_segment_size = mathutil.next_multiple(
self.tail_data_size, self._verifycap.needed_shares)
# Ciphertext hash tree root is mandatory, so that there is at most
# one ciphertext that matches this read-cap or verify-cap. The
# integrity check on the shares is not sufficient to prevent the
# original encoder from creating some shares of file A and other
# shares of file B.
self.crypttext_root_hash = d['crypttext_root_hash']
self.share_root_hash = d['share_root_hash']
# Next: things that are optional and not redundant: crypttext_hash
if d.has_key('crypttext_hash'):
self.crypttext_hash = d['crypttext_hash']
if len(self.crypttext_hash) != CRYPTO_VAL_SIZE:
raise BadURIExtension(
'crypttext_hash is required to be hashutil.CRYPTO_VAL_SIZE bytes, not %s bytes'
% (len(self.crypttext_hash), ))
# Next: things that are optional, redundant, and required to be
# consistent: codec_name, codec_params, tail_codec_params,
# num_segments, size, needed_shares, total_shares
if d.has_key('codec_name'):
if d['codec_name'] != "crs":
raise UnsupportedErasureCodec(d['codec_name'])
if d.has_key('codec_params'):
ucpss, ucpns, ucpts = codec.parse_params(d['codec_params'])
if ucpss != self.segment_size:
raise BadURIExtension("inconsistent erasure code params: "
"ucpss: %s != self.segment_size: %s" %
(ucpss, self.segment_size))
if ucpns != self._verifycap.needed_shares:
raise BadURIExtension(
"inconsistent erasure code params: ucpns: %s != "
"self._verifycap.needed_shares: %s" %
(ucpns, self._verifycap.needed_shares))
if ucpts != self._verifycap.total_shares:
raise BadURIExtension(
"inconsistent erasure code params: ucpts: %s != "
"self._verifycap.total_shares: %s" %
(ucpts, self._verifycap.total_shares))
if d.has_key('tail_codec_params'):
utcpss, utcpns, utcpts = codec.parse_params(d['tail_codec_params'])
if utcpss != self.tail_segment_size:
raise BadURIExtension(
"inconsistent erasure code params: utcpss: %s != "
"self.tail_segment_size: %s, self._verifycap.size: %s, "
"self.segment_size: %s, self._verifycap.needed_shares: %s"
% (utcpss, self.tail_segment_size, self._verifycap.size,
self.segment_size, self._verifycap.needed_shares))
if utcpns != self._verifycap.needed_shares:
raise BadURIExtension(
"inconsistent erasure code params: utcpns: %s != "
"self._verifycap.needed_shares: %s" %
(utcpns, self._verifycap.needed_shares))
if utcpts != self._verifycap.total_shares:
raise BadURIExtension(
"inconsistent erasure code params: utcpts: %s != "
"self._verifycap.total_shares: %s" %
(utcpts, self._verifycap.total_shares))
if d.has_key('num_segments'):
if d['num_segments'] != self.num_segments:
raise BadURIExtension(
"inconsistent num_segments: size: %s, "
"segment_size: %s, computed_num_segments: %s, "
"ueb_num_segments: %s" %
(self._verifycap.size, self.segment_size,
self.num_segments, d['num_segments']))
if d.has_key('size'):
if d['size'] != self._verifycap.size:
raise BadURIExtension(
"inconsistent size: URI size: %s, UEB size: %s" %
(self._verifycap.size, d['size']))
if d.has_key('needed_shares'):
if d['needed_shares'] != self._verifycap.needed_shares:
raise BadURIExtension(
"inconsistent needed shares: URI needed shares: %s, UEB "
"needed shares: %s" %
(self._verifycap.total_shares, d['needed_shares']))
if d.has_key('total_shares'):
if d['total_shares'] != self._verifycap.total_shares:
raise BadURIExtension(
"inconsistent total shares: URI total shares: %s, UEB "
"total shares: %s" %
(self._verifycap.total_shares, d['total_shares']))
# Finally, things that are deprecated and ignored: plaintext_hash,
# plaintext_root_hash
if d.get('plaintext_hash'):
log.msg(
"Found plaintext_hash in UEB. This field is deprecated for security reasons "
"and is no longer used. Ignoring. %s" % (self, ))
if d.get('plaintext_root_hash'):
log.msg(
"Found plaintext_root_hash in UEB. This field is deprecated for security "
"reasons and is no longer used. Ignoring. %s" % (self, ))
return self
def _setup_encoding_parameters(self):
"""
I set up the encoding parameters, including k, n, the number
of segments associated with this file, and the segment decoders.
"""
(seqnum,
root_hash,
IV,
segsize,
datalength,
k,
n,
known_prefix,
offsets_tuple) = self.verinfo
self._required_shares = k
self._total_shares = n
self._segment_size = segsize
self._data_length = datalength
if not IV:
self._version = MDMF_VERSION
else:
self._version = SDMF_VERSION
if datalength and segsize:
self._num_segments = mathutil.div_ceil(datalength, segsize)
self._tail_data_size = datalength % segsize
else:
self._num_segments = 0
self._tail_data_size = 0
self._segment_decoder = codec.CRSDecoder()
self._segment_decoder.set_params(segsize, k, n)
if not self._tail_data_size:
self._tail_data_size = segsize
self._tail_segment_size = mathutil.next_multiple(self._tail_data_size,
self._required_shares)
if self._tail_segment_size == self._segment_size:
self._tail_decoder = self._segment_decoder
else:
self._tail_decoder = codec.CRSDecoder()
self._tail_decoder.set_params(self._tail_segment_size,
self._required_shares,
self._total_shares)
self.log("got encoding parameters: "
"k: %d "
"n: %d "
"%d segments of %d bytes each (%d byte tail segment)" % \
(k, n, self._num_segments, self._segment_size,
self._tail_segment_size))
if self._block_hash_trees is not None:
for i in xrange(self._total_shares):
# So we don't have to do this later.
self._block_hash_trees[i] = hashtree.IncompleteHashTree(self._num_segments)
# Our last task is to tell the downloader where to start and
# where to stop. We use three parameters for that:
# - self._start_segment: the segment that we need to start
# downloading from.
# - self._current_segment: the next segment that we need to
# download.
# - self._last_segment: The last segment that we were asked to
# download.
#
# We say that the download is complete when
# self._current_segment > self._last_segment. We use
# self._start_segment and self._last_segment to know when to
# strip things off of segments, and how much to strip.
if self._offset:
self.log("got offset: %d" % self._offset)
# our start segment is the first segment containing the
# offset we were given.
start = self._offset // self._segment_size
assert start < self._num_segments
self._start_segment = start
self.log("got start segment: %d" % self._start_segment)
else:
self._start_segment = 0
# If self._read_length is None, then we want to read the whole
# file. Otherwise, we want to read only part of the file, and
# need to figure out where to stop reading.
if self._read_length is not None:
# our end segment is the last segment containing part of the
# segment that we were asked to read.
self.log("got read length %d" % self._read_length)
if self._read_length != 0:
end_data = self._offset + self._read_length
# We don't actually need to read the byte at end_data,
# but the one before it.
end = (end_data - 1) // self._segment_size
assert end < self._num_segments
self._last_segment = end
else:
self._last_segment = self._start_segment
self.log("got end segment: %d" % self._last_segment)
else:
self._last_segment = self._num_segments - 1
self._current_segment = self._start_segment
def roundup(size, blocksize=4096):
return blocksize * mathutil.div_ceil(size, blocksize)
def do_forms(self, getarg):
filled = getarg("filled", bool)
def get_and_set(name, options, default=None, astype=int):
current_value = getarg(name, astype)
i_select = T.select(name=name)
for (count, description) in options:
count = astype(count)
if ((current_value is not None and count == current_value)
or (current_value is None and count == default)):
o = T.option(value=str(count),
selected="true")[description]
else:
o = T.option(value=str(count))[description]
i_select = i_select[o]
if current_value is None:
current_value = default
return current_value, i_select
sections = {}
def add_input(section, text, entry):
if section not in sections:
sections[section] = []
sections[section].extend([T.div[text, ": ", entry], "\n"])
def add_output(section, entry):
if section not in sections:
sections[section] = []
sections[section].extend([entry, "\n"])
def build_section(section):
return T.fieldset[T.legend[section], sections[section]]
def number(value, suffix=""):
scaling = 1
if value < 1:
fmt = "%1.2g%s"
elif value < 100:
fmt = "%.1f%s"
elif value < 1000:
fmt = "%d%s"
elif value < 1e6:
fmt = "%.2fk%s"
scaling = 1e3
elif value < 1e9:
fmt = "%.2fM%s"
scaling = 1e6
elif value < 1e12:
fmt = "%.2fG%s"
scaling = 1e9
elif value < 1e15:
fmt = "%.2fT%s"
scaling = 1e12
elif value < 1e18:
fmt = "%.2fP%s"
scaling = 1e15
else:
fmt = "huge! %g%s"
return fmt % (value / scaling, suffix)
user_counts = [
(5, "5 users"),
(50, "50 users"),
(200, "200 users"),
(1000, "1k users"),
(10000, "10k users"),
(50000, "50k users"),
(100000, "100k users"),
(500000, "500k users"),
(1000000, "1M users"),
]
num_users, i_num_users = get_and_set("num_users", user_counts, 50000)
add_input("Users", "How many users are on this network?", i_num_users)
files_per_user_counts = [
(100, "100 files"),
(1000, "1k files"),
(10000, "10k files"),
(100000, "100k files"),
(1e6, "1M files"),
]
files_per_user, i_files_per_user = get_and_set("files_per_user",
files_per_user_counts,
1000)
add_input("Users", "How many files for each user? (avg)",
i_files_per_user)
space_per_user_sizes = [
(1e6, "1MB"),
(10e6, "10MB"),
(100e6, "100MB"),
(200e6, "200MB"),
(1e9, "1GB"),
(2e9, "2GB"),
(5e9, "5GB"),
(10e9, "10GB"),
(100e9, "100GB"),
(1e12, "1TB"),
]
# current allmydata average utilization 127MB per user
space_per_user, i_space_per_user = get_and_set("space_per_user",
space_per_user_sizes,
200e6)
add_input("Users", "How much data for each user? (avg)",
i_space_per_user)
sharing_ratios = [
(1.0, "1.0x"),
(1.1, "1.1x"),
(2.0, "2.0x"),
]
sharing_ratio, i_sharing_ratio = get_and_set("sharing_ratio",
sharing_ratios, 1.0,
float)
add_input(
"Users", "What is the sharing ratio? (1.0x is no-sharing and"
" no convergence)", i_sharing_ratio)
# Encoding parameters
encoding_choices = [
("3-of-10-5", "3.3x (3-of-10, repair below 5)"),
("3-of-10-8", "3.3x (3-of-10, repair below 8)"),
("5-of-10-7", "2x (5-of-10, repair below 7)"),
("8-of-10-9", "1.25x (8-of-10, repair below 9)"),
("27-of-30-28", "1.1x (27-of-30, repair below 28"),
("25-of-100-50", "4x (25-of-100, repair below 50)"),
]
encoding_parameters, i_encoding_parameters = \
get_and_set("encoding_parameters",
encoding_choices, "3-of-10-5", str)
encoding_pieces = encoding_parameters.split("-")
k = int(encoding_pieces[0])
assert encoding_pieces[1] == "of"
n = int(encoding_pieces[2])
# we repair the file when the number of available shares drops below
# this value
repair_threshold = int(encoding_pieces[3])
add_input("Servers", "What are the default encoding parameters?",
i_encoding_parameters)
# Server info
num_server_choices = [
(5, "5 servers"),
(10, "10 servers"),
(15, "15 servers"),
(30, "30 servers"),
(50, "50 servers"),
(100, "100 servers"),
(200, "200 servers"),
(300, "300 servers"),
(500, "500 servers"),
(1000, "1k servers"),
(2000, "2k servers"),
(5000, "5k servers"),
(10e3, "10k servers"),
(100e3, "100k servers"),
(1e6, "1M servers"),
]
num_servers, i_num_servers = \
get_and_set("num_servers", num_server_choices, 30, int)
add_input("Servers", "How many servers are there?", i_num_servers)
# availability is measured in dBA = -dBF, where 0dBF is 100% failure,
# 10dBF is 10% failure, 20dBF is 1% failure, etc
server_dBA_choices = [
(10, "90% [10dBA] (2.4hr/day)"),
(13, "95% [13dBA] (1.2hr/day)"),
(20, "99% [20dBA] (14min/day or 3.5days/year)"),
(23, "99.5% [23dBA] (7min/day or 1.75days/year)"),
(30, "99.9% [30dBA] (87sec/day or 9hours/year)"),
(40, "99.99% [40dBA] (60sec/week or 53min/year)"),
(50, "99.999% [50dBA] (5min per year)"),
]
server_dBA, i_server_availability = \
get_and_set("server_availability",
server_dBA_choices,
20, int)
add_input("Servers", "What is the server availability?",
i_server_availability)
drive_MTBF_choices = [
(40, "40,000 Hours"),
]
drive_MTBF, i_drive_MTBF = \
get_and_set("drive_MTBF", drive_MTBF_choices, 40, int)
add_input("Drives", "What is the hard drive MTBF?", i_drive_MTBF)
# http://www.tgdaily.com/content/view/30990/113/
# http://labs.google.com/papers/disk_failures.pdf
# google sees:
# 1.7% of the drives they replaced were 0-1 years old
# 8% of the drives they repalced were 1-2 years old
# 8.6% were 2-3 years old
# 6% were 3-4 years old, about 8% were 4-5 years old
drive_size_choices = [
(100, "100 GB"),
(250, "250 GB"),
(500, "500 GB"),
(750, "750 GB"),
]
drive_size, i_drive_size = \
get_and_set("drive_size", drive_size_choices, 750, int)
drive_size = drive_size * 1e9
add_input("Drives", "What is the capacity of each hard drive?",
i_drive_size)
drive_failure_model_choices = [
("E", "Exponential"),
("U", "Uniform"),
]
drive_failure_model, i_drive_failure_model = \
get_and_set("drive_failure_model",
drive_failure_model_choices,
"E", str)
add_input("Drives", "How should we model drive failures?",
i_drive_failure_model)
# drive_failure_rate is in failures per second
if drive_failure_model == "E":
drive_failure_rate = 1.0 / (drive_MTBF * 1000 * 3600)
else:
drive_failure_rate = 0.5 / (drive_MTBF * 1000 * 3600)
# deletion/gc/ownership mode
ownership_choices = [
("A", "no deletion, no gc, no owners"),
("B", "deletion, no gc, no owners"),
("C", "deletion, share timers, no owners"),
("D", "deletion, no gc, yes owners"),
("E", "deletion, owner timers"),
]
ownership_mode, i_ownership_mode = \
get_and_set("ownership_mode", ownership_choices,
"A", str)
add_input("Servers", "What is the ownership mode?", i_ownership_mode)
# client access behavior
access_rates = [
(1, "one file per day"),
(10, "10 files per day"),
(100, "100 files per day"),
(1000, "1k files per day"),
(10e3, "10k files per day"),
(100e3, "100k files per day"),
]
download_files_per_day, i_download_rate = \
get_and_set("download_rate", access_rates,
100, int)
add_input("Users", "How many files are downloaded per day?",
i_download_rate)
download_rate = 1.0 * download_files_per_day / (24 * 60 * 60)
upload_files_per_day, i_upload_rate = \
get_and_set("upload_rate", access_rates,
10, int)
add_input("Users", "How many files are uploaded per day?",
i_upload_rate)
upload_rate = 1.0 * upload_files_per_day / (24 * 60 * 60)
delete_files_per_day, i_delete_rate = \
get_and_set("delete_rate", access_rates,
10, int)
add_input("Users", "How many files are deleted per day?",
i_delete_rate)
delete_rate = 1.0 * delete_files_per_day / (24 * 60 * 60)
# the value is in days
lease_timers = [
(1, "one refresh per day"),
(7, "one refresh per week"),
]
lease_timer, i_lease = \
get_and_set("lease_timer", lease_timers,
7, int)
add_input(
"Users", "How frequently do clients refresh files or accounts? "
"(if necessary)", i_lease)
seconds_per_lease = 24 * 60 * 60 * lease_timer
check_timer_choices = [
(1, "every week"),
(4, "every month"),
(8, "every two months"),
(16, "every four months"),
]
check_timer, i_check_timer = \
get_and_set("check_timer", check_timer_choices, 4, int)
add_input("Users", "How frequently should we check on each file?",
i_check_timer)
file_check_interval = check_timer * 7 * 24 * 3600
if filled:
add_output("Users", T.div["Total users: %s" % number(num_users)])
add_output("Users",
T.div["Files per user: %s" % number(files_per_user)])
file_size = 1.0 * space_per_user / files_per_user
add_output("Users", T.div["Average file size: ",
number(file_size)])
total_files = num_users * files_per_user / sharing_ratio
add_output(
"Grid", T.div["Total number of files in grid: ",
number(total_files)])
total_space = num_users * space_per_user / sharing_ratio
add_output(
"Grid", T.div["Total volume of plaintext in grid: ",
number(total_space, "B")])
total_shares = n * total_files
add_output("Grid", T.div["Total shares in grid: ",
number(total_shares)])
expansion = float(n) / float(k)
total_usage = expansion * total_space
add_output("Grid", T.div["Share data in grid: ",
number(total_usage, "B")])
if n > num_servers:
# silly configuration, causes Tahoe2 to wrap and put multiple
# shares on some servers.
add_output(
"Servers", T.div["non-ideal: more shares than servers"
" (n=%d, servers=%d)" % (n, num_servers)])
# every file has at least one share on every server
buckets_per_server = total_files
shares_per_server = total_files * ((1.0 * n) / num_servers)
else:
# if nobody is full, then no lease requests will be turned
# down for lack of space, and no two shares for the same file
# will share a server. Therefore the chance that any given
# file has a share on any given server is n/num_servers.
buckets_per_server = total_files * ((1.0 * n) / num_servers)
# since each such represented file only puts one share on a
# server, the total number of shares per server is the same.
shares_per_server = buckets_per_server
add_output(
"Servers", T.div["Buckets per server: ",
number(buckets_per_server)])
add_output("Servers", T.div["Shares per server: ",
number(shares_per_server)])
# how much space is used on the storage servers for the shares?
# the share data itself
share_data_per_server = total_usage / num_servers
add_output(
"Servers", T.div["Share data per server: ",
number(share_data_per_server, "B")])
# this is determined empirically. H=hashsize=32, for a one-segment
# file and 3-of-10 encoding
share_validation_per_server = 266 * shares_per_server
# this could be 423*buckets_per_server, if we moved the URI
# extension into a separate file, but that would actually consume
# *more* space (minimum filesize is 4KiB), unless we moved all
# shares for a given bucket into a single file.
share_uri_extension_per_server = 423 * shares_per_server
# ownership mode adds per-bucket data
H = 32 # depends upon the desired security of delete/refresh caps
# bucket_lease_size is the amount of data needed to keep track of
# the delete/refresh caps for each bucket.
bucket_lease_size = 0
client_bucket_refresh_rate = 0
owner_table_size = 0
if ownership_mode in ("B", "C", "D", "E"):
bucket_lease_size = sharing_ratio * 1.0 * H
if ownership_mode in ("B", "C"):
# refreshes per second per client
client_bucket_refresh_rate = (1.0 * n * files_per_user /
seconds_per_lease)
add_output(
"Users", T.div["Client share refresh rate (outbound): ",
number(client_bucket_refresh_rate, "Hz")])
server_bucket_refresh_rate = (client_bucket_refresh_rate *
num_users / num_servers)
add_output(
"Servers", T.div["Server share refresh rate (inbound): ",
number(server_bucket_refresh_rate, "Hz")])
if ownership_mode in ("D", "E"):
# each server must maintain a bidirectional mapping from
# buckets to owners. One way to implement this would be to
# put a list of four-byte owner numbers into each bucket, and
# a list of four-byte share numbers into each owner (although
# of course we'd really just throw it into a database and let
# the experts take care of the details).
owner_table_size = 2 * (buckets_per_server * sharing_ratio * 4)
if ownership_mode in ("E", ):
# in this mode, clients must refresh one timer per server
client_account_refresh_rate = (1.0 * num_servers /
seconds_per_lease)
add_output(
"Users", T.div["Client account refresh rate (outbound): ",
number(client_account_refresh_rate, "Hz")])
server_account_refresh_rate = (client_account_refresh_rate *
num_users / num_servers)
add_output(
"Servers",
T.div["Server account refresh rate (inbound): ",
number(server_account_refresh_rate, "Hz")])
# TODO: buckets vs shares here is a bit wonky, but in
# non-wrapping grids it shouldn't matter
share_lease_per_server = bucket_lease_size * buckets_per_server
share_ownertable_per_server = owner_table_size
share_space_per_server = (share_data_per_server +
share_validation_per_server +
share_uri_extension_per_server +
share_lease_per_server +
share_ownertable_per_server)
add_output(
"Servers",
T.div["Share space per server: ",
number(share_space_per_server, "B"), " (data ",
number(share_data_per_server, "B"), ", validation ",
number(share_validation_per_server, "B"), ", UEB ",
number(share_uri_extension_per_server, "B"), ", lease ",
number(share_lease_per_server, "B"), ", ownertable ",
number(share_ownertable_per_server, "B"), ")", ])
# rates
client_download_share_rate = download_rate * k
client_download_byte_rate = download_rate * file_size
add_output(
"Users", T.div["download rate: shares = ",
number(client_download_share_rate, "Hz"),
" , bytes = ",
number(client_download_byte_rate, "Bps"), ])
total_file_check_rate = 1.0 * total_files / file_check_interval
client_check_share_rate = total_file_check_rate / num_users
add_output(
"Users", T.div["file check rate: shares = ",
number(client_check_share_rate, "Hz"),
" (interval = %s)" %
number(1 / client_check_share_rate, "s"), ])
client_upload_share_rate = upload_rate * n
# TODO: doesn't include overhead
client_upload_byte_rate = upload_rate * file_size * expansion
add_output(
"Users", T.div["upload rate: shares = ",
number(client_upload_share_rate, "Hz"),
" , bytes = ",
number(client_upload_byte_rate, "Bps"), ])
client_delete_share_rate = delete_rate * n
server_inbound_share_rate = (client_upload_share_rate * num_users /
num_servers)
server_inbound_byte_rate = (client_upload_byte_rate * num_users /
num_servers)
add_output(
"Servers", T.div["upload rate (inbound): shares = ",
number(server_inbound_share_rate, "Hz"),
" , bytes = ",
number(server_inbound_byte_rate, "Bps"), ])
add_output(
"Servers",
T.div["share check rate (inbound): ",
number(total_file_check_rate * n / num_servers, "Hz"), ])
server_share_modify_rate = (
(client_upload_share_rate + client_delete_share_rate) *
num_users / num_servers)
add_output(
"Servers", T.div["share modify rate: shares = ",
number(server_share_modify_rate, "Hz"), ])
server_outbound_share_rate = (client_download_share_rate *
num_users / num_servers)
server_outbound_byte_rate = (client_download_byte_rate *
num_users / num_servers)
add_output(
"Servers", T.div["download rate (outbound): shares = ",
number(server_outbound_share_rate, "Hz"),
" , bytes = ",
number(server_outbound_byte_rate, "Bps"), ])
total_share_space = num_servers * share_space_per_server
add_output(
"Grid", T.div["Share space consumed: ",
number(total_share_space, "B")])
add_output(
"Grid", T.div[" %% validation: %.2f%%" %
(100.0 * share_validation_per_server /
share_space_per_server)])
add_output(
"Grid", T.div[" %% uri-extension: %.2f%%" %
(100.0 * share_uri_extension_per_server /
share_space_per_server)])
add_output(
"Grid", T.div[" %% lease data: %.2f%%" %
(100.0 * share_lease_per_server /
share_space_per_server)])
add_output(
"Grid", T.div[" %% owner data: %.2f%%" %
(100.0 * share_ownertable_per_server /
share_space_per_server)])
add_output(
"Grid", T.div[" %% share data: %.2f%%" %
(100.0 * share_data_per_server /
share_space_per_server)])
add_output(
"Grid", T.div["file check rate: ",
number(total_file_check_rate, "Hz")])
total_drives = max(
mathutil.div_ceil(int(total_share_space), int(drive_size)),
num_servers)
add_output(
"Drives", T.div["Total drives: ",
number(total_drives), " drives"])
drives_per_server = mathutil.div_ceil(total_drives, num_servers)
add_output("Servers", T.div["Drives per server: ",
drives_per_server])
# costs
if drive_size == 750 * 1e9:
add_output("Servers", T.div["750GB drive: $250 each"])
drive_cost = 250
else:
add_output("Servers",
T.div[T.b["unknown cost per drive, assuming $100"]])
drive_cost = 100
if drives_per_server <= 4:
add_output("Servers", T.div["1U box with <= 4 drives: $1500"])
server_cost = 1500 # typical 1U box
elif drives_per_server <= 12:
add_output("Servers", T.div["2U box with <= 12 drives: $2500"])
server_cost = 2500 # 2U box
else:
add_output(
"Servers", T.div[T.b["Note: too many drives per server, "
"assuming $3000"]])
server_cost = 3000
server_capital_cost = (server_cost +
drives_per_server * drive_cost)
total_server_cost = float(num_servers * server_capital_cost)
add_output(
"Servers", T.div["Capital cost per server: $",
server_capital_cost])
add_output(
"Grid", T.div["Capital cost for all servers: $",
number(total_server_cost)])
# $70/Mbps/mo
# $44/server/mo power+space
server_bandwidth = max(server_inbound_byte_rate,
server_outbound_byte_rate)
server_bandwidth_mbps = mathutil.div_ceil(
int(server_bandwidth * 8), int(1e6))
server_monthly_cost = 70 * server_bandwidth_mbps + 44
add_output(
"Servers", T.div["Monthly cost per server: $",
server_monthly_cost])
add_output(
"Users", T.div["Capital cost per user: $",
number(total_server_cost / num_users)])
# reliability
any_drive_failure_rate = total_drives * drive_failure_rate
any_drive_MTBF = 1 // any_drive_failure_rate # in seconds
any_drive_MTBF_days = any_drive_MTBF / 86400
add_output(
"Drives", T.div["MTBF (any drive): ",
number(any_drive_MTBF_days), " days"])
drive_replacement_monthly_cost = (float(drive_cost) *
any_drive_failure_rate * 30 *
86400)
add_output(
"Grid", T.div["Monthly cost of replacing drives: $",
number(drive_replacement_monthly_cost)])
total_server_monthly_cost = float(num_servers *
server_monthly_cost +
drive_replacement_monthly_cost)
add_output(
"Grid", T.div["Monthly cost for all servers: $",
number(total_server_monthly_cost)])
add_output(
"Users", T.div["Monthly cost per user: $",
number(total_server_monthly_cost / num_users)])
# availability
file_dBA = self.file_availability(k, n, server_dBA)
user_files_dBA = self.many_files_availability(
file_dBA, files_per_user)
all_files_dBA = self.many_files_availability(file_dBA, total_files)
add_output(
"Users",
T.div["availability of: ",
"arbitrary file = %d dBA, " % file_dBA,
"all files of user1 = %d dBA, " % user_files_dBA,
"all files in grid = %d dBA" % all_files_dBA, ],
)
time_until_files_lost = (n - k + 1) / any_drive_failure_rate
add_output(
"Grid",
T.div["avg time until files are lost: ",
number(time_until_files_lost, "s"), ", ",
number(time_until_files_lost / 86400, " days"), ])
share_data_loss_rate = any_drive_failure_rate * drive_size
add_output(
"Grid", T.div["share data loss rate: ",
number(share_data_loss_rate, "Bps")])
# the worst-case survival numbers occur when we do a file check
# and the file is just above the threshold for repair (so we
# decide to not repair it). The question is then: what is the
# chance that the file will decay so badly before the next check
# that we can't recover it? The resulting probability is per
# check interval.
# Note that the chances of us getting into this situation are low.
P_disk_failure_during_interval = (drive_failure_rate *
file_check_interval)
disk_failure_dBF = 10 * math.log10(P_disk_failure_during_interval)
disk_failure_dBA = -disk_failure_dBF
file_survives_dBA = self.file_availability(k, repair_threshold,
disk_failure_dBA)
user_files_survives_dBA = self.many_files_availability( \
file_survives_dBA, files_per_user)
all_files_survives_dBA = self.many_files_availability( \
file_survives_dBA, total_files)
add_output(
"Users",
T.div["survival of: ",
"arbitrary file = %d dBA, " % file_survives_dBA,
"all files of user1 = %d dBA, " %
user_files_survives_dBA,
"all files in grid = %d dBA" % all_files_survives_dBA,
" (per worst-case check interval)", ])
all_sections = []
all_sections.append(build_section("Users"))
all_sections.append(build_section("Servers"))
all_sections.append(build_section("Drives"))
if "Grid" in sections:
all_sections.append(build_section("Grid"))
f = T.form(action=".", method="post", enctype="multipart/form-data")
if filled:
action = "Recompute"
else:
action = "Compute"
f = f[T.input(type="hidden", name="filled", value="true"),
T.input(type="submit", value=action), all_sections, ]
try:
from allmydata import reliability
# we import this just to test to see if the page is available
_hush_pyflakes = reliability
del _hush_pyflakes
f = [T.div[T.a(href="../reliability")["Reliability Math"]], f]
except ImportError:
pass
return f
def do_forms(self, getarg):
filled = getarg("filled", bool)
def get_and_set(name, options, default=None, astype=int):
current_value = getarg(name, astype)
i_select = T.select(name=name)
for (count, description) in options:
count = astype(count)
if ((current_value is not None and count == current_value) or
(current_value is None and count == default)):
o = T.option(value=str(count), selected="true")[description]
else:
o = T.option(value=str(count))[description]
i_select = i_select[o]
if current_value is None:
current_value = default
return current_value, i_select
sections = {}
def add_input(section, text, entry):
if section not in sections:
sections[section] = []
sections[section].extend([T.div[text, ": ", entry], "\n"])
def add_output(section, entry):
if section not in sections:
sections[section] = []
sections[section].extend([entry, "\n"])
def build_section(section):
return T.fieldset[T.legend[section], sections[section]]
def number(value, suffix=""):
scaling = 1
if value < 1:
fmt = "%1.2g%s"
elif value < 100:
fmt = "%.1f%s"
elif value < 1000:
fmt = "%d%s"
elif value < 1e6:
fmt = "%.2fk%s"; scaling = 1e3
elif value < 1e9:
fmt = "%.2fM%s"; scaling = 1e6
elif value < 1e12:
fmt = "%.2fG%s"; scaling = 1e9
elif value < 1e15:
fmt = "%.2fT%s"; scaling = 1e12
elif value < 1e18:
fmt = "%.2fP%s"; scaling = 1e15
else:
fmt = "huge! %g%s"
return fmt % (value / scaling, suffix)
user_counts = [(5, "5 users"),
(50, "50 users"),
(200, "200 users"),
(1000, "1k users"),
(10000, "10k users"),
(50000, "50k users"),
(100000, "100k users"),
(500000, "500k users"),
(1000000, "1M users"),
]
num_users, i_num_users = get_and_set("num_users", user_counts, 50000)
add_input("Users",
"How many users are on this network?", i_num_users)
files_per_user_counts = [(100, "100 files"),
(1000, "1k files"),
(10000, "10k files"),
(100000, "100k files"),
(1e6, "1M files"),
]
files_per_user, i_files_per_user = get_and_set("files_per_user",
files_per_user_counts,
1000)
add_input("Users",
"How many files for each user? (avg)",
i_files_per_user)
space_per_user_sizes = [(1e6, "1MB"),
(10e6, "10MB"),
(100e6, "100MB"),
(200e6, "200MB"),
(1e9, "1GB"),
(2e9, "2GB"),
(5e9, "5GB"),
(10e9, "10GB"),
(100e9, "100GB"),
(1e12, "1TB"),
]
# current allmydata average utilization 127MB per user
space_per_user, i_space_per_user = get_and_set("space_per_user",
space_per_user_sizes,
200e6)
add_input("Users",
"How much data for each user? (avg)",
i_space_per_user)
sharing_ratios = [(1.0, "1.0x"),
(1.1, "1.1x"),
(2.0, "2.0x"),
]
sharing_ratio, i_sharing_ratio = get_and_set("sharing_ratio",
sharing_ratios, 1.0,
float)
add_input("Users",
"What is the sharing ratio? (1.0x is no-sharing and"
" no convergence)", i_sharing_ratio)
# Encoding parameters
encoding_choices = [("3-of-10-5", "3.3x (3-of-10, repair below 5)"),
("3-of-10-8", "3.3x (3-of-10, repair below 8)"),
("5-of-10-7", "2x (5-of-10, repair below 7)"),
("8-of-10-9", "1.25x (8-of-10, repair below 9)"),
("27-of-30-28", "1.1x (27-of-30, repair below 28"),
("25-of-100-50", "4x (25-of-100, repair below 50)"),
]
encoding_parameters, i_encoding_parameters = \
get_and_set("encoding_parameters",
encoding_choices, "3-of-10-5", str)
encoding_pieces = encoding_parameters.split("-")
k = int(encoding_pieces[0])
assert encoding_pieces[1] == "of"
n = int(encoding_pieces[2])
# we repair the file when the number of available shares drops below
# this value
repair_threshold = int(encoding_pieces[3])
add_input("Servers",
"What are the default encoding parameters?",
i_encoding_parameters)
# Server info
num_server_choices = [ (5, "5 servers"),
(10, "10 servers"),
(15, "15 servers"),
(30, "30 servers"),
(50, "50 servers"),
(100, "100 servers"),
(200, "200 servers"),
(300, "300 servers"),
(500, "500 servers"),
(1000, "1k servers"),
(2000, "2k servers"),
(5000, "5k servers"),
(10e3, "10k servers"),
(100e3, "100k servers"),
(1e6, "1M servers"),
]
num_servers, i_num_servers = \
get_and_set("num_servers", num_server_choices, 30, int)
add_input("Servers",
"How many servers are there?", i_num_servers)
# availability is measured in dBA = -dBF, where 0dBF is 100% failure,
# 10dBF is 10% failure, 20dBF is 1% failure, etc
server_dBA_choices = [ (10, "90% [10dBA] (2.4hr/day)"),
(13, "95% [13dBA] (1.2hr/day)"),
(20, "99% [20dBA] (14min/day or 3.5days/year)"),
(23, "99.5% [23dBA] (7min/day or 1.75days/year)"),
(30, "99.9% [30dBA] (87sec/day or 9hours/year)"),
(40, "99.99% [40dBA] (60sec/week or 53min/year)"),
(50, "99.999% [50dBA] (5min per year)"),
]
server_dBA, i_server_availability = \
get_and_set("server_availability",
server_dBA_choices,
20, int)
add_input("Servers",
"What is the server availability?", i_server_availability)
drive_MTBF_choices = [ (40, "40,000 Hours"),
]
drive_MTBF, i_drive_MTBF = \
get_and_set("drive_MTBF", drive_MTBF_choices, 40, int)
add_input("Drives",
"What is the hard drive MTBF?", i_drive_MTBF)
# http://www.tgdaily.com/content/view/30990/113/
# http://labs.google.com/papers/disk_failures.pdf
# google sees:
# 1.7% of the drives they replaced were 0-1 years old
# 8% of the drives they repalced were 1-2 years old
# 8.6% were 2-3 years old
# 6% were 3-4 years old, about 8% were 4-5 years old
drive_size_choices = [ (100, "100 GB"),
(250, "250 GB"),
(500, "500 GB"),
(750, "750 GB"),
]
drive_size, i_drive_size = \
get_and_set("drive_size", drive_size_choices, 750, int)
drive_size = drive_size * 1e9
add_input("Drives",
"What is the capacity of each hard drive?", i_drive_size)
drive_failure_model_choices = [ ("E", "Exponential"),
("U", "Uniform"),
]
drive_failure_model, i_drive_failure_model = \
get_and_set("drive_failure_model",
drive_failure_model_choices,
"E", str)
add_input("Drives",
"How should we model drive failures?", i_drive_failure_model)
# drive_failure_rate is in failures per second
if drive_failure_model == "E":
drive_failure_rate = 1.0 / (drive_MTBF * 1000 * 3600)
else:
drive_failure_rate = 0.5 / (drive_MTBF * 1000 * 3600)
# deletion/gc/ownership mode
ownership_choices = [ ("A", "no deletion, no gc, no owners"),
("B", "deletion, no gc, no owners"),
("C", "deletion, share timers, no owners"),
("D", "deletion, no gc, yes owners"),
("E", "deletion, owner timers"),
]
ownership_mode, i_ownership_mode = \
get_and_set("ownership_mode", ownership_choices,
"A", str)
add_input("Servers",
"What is the ownership mode?", i_ownership_mode)
# client access behavior
access_rates = [ (1, "one file per day"),
(10, "10 files per day"),
(100, "100 files per day"),
(1000, "1k files per day"),
(10e3, "10k files per day"),
(100e3, "100k files per day"),
]
download_files_per_day, i_download_rate = \
get_and_set("download_rate", access_rates,
100, int)
add_input("Users",
"How many files are downloaded per day?", i_download_rate)
download_rate = 1.0 * download_files_per_day / (24*60*60)
upload_files_per_day, i_upload_rate = \
get_and_set("upload_rate", access_rates,
10, int)
add_input("Users",
"How many files are uploaded per day?", i_upload_rate)
upload_rate = 1.0 * upload_files_per_day / (24*60*60)
delete_files_per_day, i_delete_rate = \
get_and_set("delete_rate", access_rates,
10, int)
add_input("Users",
"How many files are deleted per day?", i_delete_rate)
delete_rate = 1.0 * delete_files_per_day / (24*60*60)
# the value is in days
lease_timers = [ (1, "one refresh per day"),
(7, "one refresh per week"),
]
lease_timer, i_lease = \
get_and_set("lease_timer", lease_timers,
7, int)
add_input("Users",
"How frequently do clients refresh files or accounts? "
"(if necessary)",
i_lease)
seconds_per_lease = 24*60*60*lease_timer
check_timer_choices = [ (1, "every week"),
(4, "every month"),
(8, "every two months"),
(16, "every four months"),
]
check_timer, i_check_timer = \
get_and_set("check_timer", check_timer_choices, 4, int)
add_input("Users",
"How frequently should we check on each file?",
i_check_timer)
file_check_interval = check_timer * 7 * 24 * 3600
if filled:
add_output("Users", T.div["Total users: %s" % number(num_users)])
add_output("Users",
T.div["Files per user: %s" % number(files_per_user)])
file_size = 1.0 * space_per_user / files_per_user
add_output("Users",
T.div["Average file size: ", number(file_size)])
total_files = num_users * files_per_user / sharing_ratio
add_output("Grid",
T.div["Total number of files in grid: ",
number(total_files)])
total_space = num_users * space_per_user / sharing_ratio
add_output("Grid",
T.div["Total volume of plaintext in grid: ",
number(total_space, "B")])
total_shares = n * total_files
add_output("Grid",
T.div["Total shares in grid: ", number(total_shares)])
expansion = float(n) / float(k)
total_usage = expansion * total_space
add_output("Grid",
T.div["Share data in grid: ", number(total_usage, "B")])
if n > num_servers:
# silly configuration, causes Tahoe2 to wrap and put multiple
# shares on some servers.
add_output("Servers",
T.div["non-ideal: more shares than servers"
" (n=%d, servers=%d)" % (n, num_servers)])
# every file has at least one share on every server
buckets_per_server = total_files
shares_per_server = total_files * ((1.0 * n) / num_servers)
else:
# if nobody is full, then no lease requests will be turned
# down for lack of space, and no two shares for the same file
# will share a server. Therefore the chance that any given
# file has a share on any given server is n/num_servers.
buckets_per_server = total_files * ((1.0 * n) / num_servers)
# since each such represented file only puts one share on a
# server, the total number of shares per server is the same.
shares_per_server = buckets_per_server
add_output("Servers",
T.div["Buckets per server: ",
number(buckets_per_server)])
add_output("Servers",
T.div["Shares per server: ",
number(shares_per_server)])
# how much space is used on the storage servers for the shares?
# the share data itself
share_data_per_server = total_usage / num_servers
add_output("Servers",
T.div["Share data per server: ",
number(share_data_per_server, "B")])
# this is determined empirically. H=hashsize=32, for a one-segment
# file and 3-of-10 encoding
share_validation_per_server = 266 * shares_per_server
# this could be 423*buckets_per_server, if we moved the URI
# extension into a separate file, but that would actually consume
# *more* space (minimum filesize is 4KiB), unless we moved all
# shares for a given bucket into a single file.
share_uri_extension_per_server = 423 * shares_per_server
# ownership mode adds per-bucket data
H = 32 # depends upon the desired security of delete/refresh caps
# bucket_lease_size is the amount of data needed to keep track of
# the delete/refresh caps for each bucket.
bucket_lease_size = 0
client_bucket_refresh_rate = 0
owner_table_size = 0
if ownership_mode in ("B", "C", "D", "E"):
bucket_lease_size = sharing_ratio * 1.0 * H
if ownership_mode in ("B", "C"):
# refreshes per second per client
client_bucket_refresh_rate = (1.0 * n * files_per_user /
seconds_per_lease)
add_output("Users",
T.div["Client share refresh rate (outbound): ",
number(client_bucket_refresh_rate, "Hz")])
server_bucket_refresh_rate = (client_bucket_refresh_rate *
num_users / num_servers)
add_output("Servers",
T.div["Server share refresh rate (inbound): ",
number(server_bucket_refresh_rate, "Hz")])
if ownership_mode in ("D", "E"):
# each server must maintain a bidirectional mapping from
# buckets to owners. One way to implement this would be to
# put a list of four-byte owner numbers into each bucket, and
# a list of four-byte share numbers into each owner (although
# of course we'd really just throw it into a database and let
# the experts take care of the details).
owner_table_size = 2*(buckets_per_server * sharing_ratio * 4)
if ownership_mode in ("E",):
# in this mode, clients must refresh one timer per server
client_account_refresh_rate = (1.0 * num_servers /
seconds_per_lease)
add_output("Users",
T.div["Client account refresh rate (outbound): ",
number(client_account_refresh_rate, "Hz")])
server_account_refresh_rate = (client_account_refresh_rate *
num_users / num_servers)
add_output("Servers",
T.div["Server account refresh rate (inbound): ",
number(server_account_refresh_rate, "Hz")])
# TODO: buckets vs shares here is a bit wonky, but in
# non-wrapping grids it shouldn't matter
share_lease_per_server = bucket_lease_size * buckets_per_server
share_ownertable_per_server = owner_table_size
share_space_per_server = (share_data_per_server +
share_validation_per_server +
share_uri_extension_per_server +
share_lease_per_server +
share_ownertable_per_server)
add_output("Servers",
T.div["Share space per server: ",
number(share_space_per_server, "B"),
" (data ",
number(share_data_per_server, "B"),
", validation ",
number(share_validation_per_server, "B"),
", UEB ",
number(share_uri_extension_per_server, "B"),
", lease ",
number(share_lease_per_server, "B"),
", ownertable ",
number(share_ownertable_per_server, "B"),
")",
])
# rates
client_download_share_rate = download_rate * k
client_download_byte_rate = download_rate * file_size
add_output("Users",
T.div["download rate: shares = ",
number(client_download_share_rate, "Hz"),
" , bytes = ",
number(client_download_byte_rate, "Bps"),
])
total_file_check_rate = 1.0 * total_files / file_check_interval
client_check_share_rate = total_file_check_rate / num_users
add_output("Users",
T.div["file check rate: shares = ",
number(client_check_share_rate, "Hz"),
" (interval = %s)" %
number(1 / client_check_share_rate, "s"),
])
client_upload_share_rate = upload_rate * n
# TODO: doesn't include overhead
client_upload_byte_rate = upload_rate * file_size * expansion
add_output("Users",
T.div["upload rate: shares = ",
number(client_upload_share_rate, "Hz"),
" , bytes = ",
number(client_upload_byte_rate, "Bps"),
])
client_delete_share_rate = delete_rate * n
server_inbound_share_rate = (client_upload_share_rate *
num_users / num_servers)
server_inbound_byte_rate = (client_upload_byte_rate *
num_users / num_servers)
add_output("Servers",
T.div["upload rate (inbound): shares = ",
number(server_inbound_share_rate, "Hz"),
" , bytes = ",
number(server_inbound_byte_rate, "Bps"),
])
add_output("Servers",
T.div["share check rate (inbound): ",
number(total_file_check_rate * n / num_servers,
"Hz"),
])
server_share_modify_rate = ((client_upload_share_rate +
client_delete_share_rate) *
num_users / num_servers)
add_output("Servers",
T.div["share modify rate: shares = ",
number(server_share_modify_rate, "Hz"),
])
server_outbound_share_rate = (client_download_share_rate *
num_users / num_servers)
server_outbound_byte_rate = (client_download_byte_rate *
num_users / num_servers)
add_output("Servers",
T.div["download rate (outbound): shares = ",
number(server_outbound_share_rate, "Hz"),
" , bytes = ",
number(server_outbound_byte_rate, "Bps"),
])
total_share_space = num_servers * share_space_per_server
add_output("Grid",
T.div["Share space consumed: ",
number(total_share_space, "B")])
add_output("Grid",
T.div[" %% validation: %.2f%%" %
(100.0 * share_validation_per_server /
share_space_per_server)])
add_output("Grid",
T.div[" %% uri-extension: %.2f%%" %
(100.0 * share_uri_extension_per_server /
share_space_per_server)])
add_output("Grid",
T.div[" %% lease data: %.2f%%" %
(100.0 * share_lease_per_server /
share_space_per_server)])
add_output("Grid",
T.div[" %% owner data: %.2f%%" %
(100.0 * share_ownertable_per_server /
share_space_per_server)])
add_output("Grid",
T.div[" %% share data: %.2f%%" %
(100.0 * share_data_per_server /
share_space_per_server)])
add_output("Grid",
T.div["file check rate: ",
number(total_file_check_rate,
"Hz")])
total_drives = max(mathutil.div_ceil(int(total_share_space),
int(drive_size)),
num_servers)
add_output("Drives",
T.div["Total drives: ", number(total_drives), " drives"])
drives_per_server = mathutil.div_ceil(total_drives, num_servers)
add_output("Servers",
T.div["Drives per server: ", drives_per_server])
# costs
if drive_size == 750 * 1e9:
add_output("Servers", T.div["750GB drive: $250 each"])
drive_cost = 250
else:
add_output("Servers",
T.div[T.b["unknown cost per drive, assuming $100"]])
drive_cost = 100
if drives_per_server <= 4:
add_output("Servers", T.div["1U box with <= 4 drives: $1500"])
server_cost = 1500 # typical 1U box
elif drives_per_server <= 12:
add_output("Servers", T.div["2U box with <= 12 drives: $2500"])
server_cost = 2500 # 2U box
else:
add_output("Servers",
T.div[T.b["Note: too many drives per server, "
"assuming $3000"]])
server_cost = 3000
server_capital_cost = (server_cost + drives_per_server * drive_cost)
total_server_cost = float(num_servers * server_capital_cost)
add_output("Servers", T.div["Capital cost per server: $",
server_capital_cost])
add_output("Grid", T.div["Capital cost for all servers: $",
number(total_server_cost)])
# $70/Mbps/mo
# $44/server/mo power+space
server_bandwidth = max(server_inbound_byte_rate,
server_outbound_byte_rate)
server_bandwidth_mbps = mathutil.div_ceil(int(server_bandwidth*8),
int(1e6))
server_monthly_cost = 70*server_bandwidth_mbps + 44
add_output("Servers", T.div["Monthly cost per server: $",
server_monthly_cost])
add_output("Users", T.div["Capital cost per user: $",
number(total_server_cost / num_users)])
# reliability
any_drive_failure_rate = total_drives * drive_failure_rate
any_drive_MTBF = 1 // any_drive_failure_rate # in seconds
any_drive_MTBF_days = any_drive_MTBF / 86400
add_output("Drives",
T.div["MTBF (any drive): ",
number(any_drive_MTBF_days), " days"])
drive_replacement_monthly_cost = (float(drive_cost)
* any_drive_failure_rate
*30*86400)
add_output("Grid",
T.div["Monthly cost of replacing drives: $",
number(drive_replacement_monthly_cost)])
total_server_monthly_cost = float(num_servers * server_monthly_cost
+ drive_replacement_monthly_cost)
add_output("Grid", T.div["Monthly cost for all servers: $",
number(total_server_monthly_cost)])
add_output("Users",
T.div["Monthly cost per user: $",
number(total_server_monthly_cost / num_users)])
# availability
file_dBA = self.file_availability(k, n, server_dBA)
user_files_dBA = self.many_files_availability(file_dBA,
files_per_user)
all_files_dBA = self.many_files_availability(file_dBA, total_files)
add_output("Users",
T.div["availability of: ",
"arbitrary file = %d dBA, " % file_dBA,
"all files of user1 = %d dBA, " % user_files_dBA,
"all files in grid = %d dBA" % all_files_dBA,
],
)
time_until_files_lost = (n-k+1) / any_drive_failure_rate
add_output("Grid",
T.div["avg time until files are lost: ",
number(time_until_files_lost, "s"), ", ",
number(time_until_files_lost/86400, " days"),
])
share_data_loss_rate = any_drive_failure_rate * drive_size
add_output("Grid",
T.div["share data loss rate: ",
number(share_data_loss_rate,"Bps")])
# the worst-case survival numbers occur when we do a file check
# and the file is just above the threshold for repair (so we
# decide to not repair it). The question is then: what is the
# chance that the file will decay so badly before the next check
# that we can't recover it? The resulting probability is per
# check interval.
# Note that the chances of us getting into this situation are low.
P_disk_failure_during_interval = (drive_failure_rate *
file_check_interval)
disk_failure_dBF = 10*math.log10(P_disk_failure_during_interval)
disk_failure_dBA = -disk_failure_dBF
file_survives_dBA = self.file_availability(k, repair_threshold,
disk_failure_dBA)
user_files_survives_dBA = self.many_files_availability( \
file_survives_dBA, files_per_user)
all_files_survives_dBA = self.many_files_availability( \
file_survives_dBA, total_files)
add_output("Users",
T.div["survival of: ",
"arbitrary file = %d dBA, " % file_survives_dBA,
"all files of user1 = %d dBA, " %
user_files_survives_dBA,
"all files in grid = %d dBA" %
all_files_survives_dBA,
" (per worst-case check interval)",
])
all_sections = []
all_sections.append(build_section("Users"))
all_sections.append(build_section("Servers"))
all_sections.append(build_section("Drives"))
if "Grid" in sections:
all_sections.append(build_section("Grid"))
f = T.form(action=".", method="post", enctype="multipart/form-data")
if filled:
action = "Recompute"
else:
action = "Compute"
f = f[T.input(type="hidden", name="filled", value="true"),
T.input(type="submit", value=action),
all_sections,
]
try:
from allmydata import reliability
# we import this just to test to see if the page is available
_hush_pyflakes = reliability
del _hush_pyflakes
f = [T.div[T.a(href="../reliability")["Reliability Math"]], f]
except ImportError:
pass
return f
def _setup_encoding_parameters(self):
"""
I set up the encoding parameters, including k, n, the number
of segments associated with this file, and the segment decoders.
"""
(seqnum, root_hash, IV, segsize, datalength, k, n, known_prefix,
offsets_tuple) = self.verinfo
self._required_shares = k
self._total_shares = n
self._segment_size = segsize
self._data_length = datalength
if not IV:
self._version = MDMF_VERSION
else:
self._version = SDMF_VERSION
if datalength and segsize:
self._num_segments = mathutil.div_ceil(datalength, segsize)
self._tail_data_size = datalength % segsize
else:
self._num_segments = 0
self._tail_data_size = 0
self._segment_decoder = codec.CRSDecoder()
self._segment_decoder.set_params(segsize, k, n)
if not self._tail_data_size:
self._tail_data_size = segsize
self._tail_segment_size = mathutil.next_multiple(
self._tail_data_size, self._required_shares)
if self._tail_segment_size == self._segment_size:
self._tail_decoder = self._segment_decoder
else:
self._tail_decoder = codec.CRSDecoder()
self._tail_decoder.set_params(self._tail_segment_size,
self._required_shares,
self._total_shares)
self.log("got encoding parameters: "
"k: %d "
"n: %d "
"%d segments of %d bytes each (%d byte tail segment)" % \
(k, n, self._num_segments, self._segment_size,
self._tail_segment_size))
# Our last task is to tell the downloader where to start and
# where to stop. We use three parameters for that:
# - self._start_segment: the segment that we need to start
# downloading from.
# - self._current_segment: the next segment that we need to
# download.
# - self._last_segment: The last segment that we were asked to
# download.
#
# We say that the download is complete when
# self._current_segment > self._last_segment. We use
# self._start_segment and self._last_segment to know when to
# strip things off of segments, and how much to strip.
if self._offset:
self.log("got offset: %d" % self._offset)
# our start segment is the first segment containing the
# offset we were given.
start = self._offset // self._segment_size
assert start < self._num_segments
self._start_segment = start
self.log("got start segment: %d" % self._start_segment)
else:
self._start_segment = 0
# If self._read_length is None, then we want to read the whole
# file. Otherwise, we want to read only part of the file, and
# need to figure out where to stop reading.
if self._read_length is not None:
# our end segment is the last segment containing part of the
# segment that we were asked to read.
self.log("got read length %d" % self._read_length)
if self._read_length != 0:
end_data = self._offset + self._read_length
# We don't actually need to read the byte at end_data,
# but the one before it.
end = (end_data - 1) // self._segment_size
assert end < self._num_segments
self._last_segment = end
else:
self._last_segment = self._start_segment
self.log("got end segment: %d" % self._last_segment)
else:
self._last_segment = self._num_segments - 1
self._current_segment = self._start_segment
def do_test(self, size, required_shares, max_shares, fewer_shares=None):
data0s = [os.urandom(mathutil.div_ceil(size, required_shares)) for i in range(required_shares)]
enc = CRSEncoder()
enc.set_params(size, required_shares, max_shares)
params = enc.get_params()
assert params == (size, required_shares, max_shares)
serialized_params = enc.get_serialized_params()
self.assertEqual(parse_params(serialized_params), params)
log.msg("params: %s" % (params,))
d = enc.encode(data0s)
def _done_encoding_all(shares_and_shareids):
(shares, shareids) = shares_and_shareids
self.failUnlessEqual(len(shares), max_shares)
self.shares = shares
self.shareids = shareids
d.addCallback(_done_encoding_all)
if fewer_shares is not None:
# also validate that the desired_shareids= parameter works
desired_shareids = random.sample(list(range(max_shares)), fewer_shares)
d.addCallback(lambda res: enc.encode(data0s, desired_shareids))
def _check_fewer_shares(some_shares_and_their_shareids):
(some_shares, their_shareids) = some_shares_and_their_shareids
self.failUnlessEqual(tuple(their_shareids), tuple(desired_shareids))
d.addCallback(_check_fewer_shares)
def _decode(shares_and_shareids):
(shares, shareids) = shares_and_shareids
dec = CRSDecoder()
dec.set_params(*params)
d1 = dec.decode(shares, shareids)
return d1
def _check_data(decoded_shares):
self.failUnlessEqual(len(b''.join(decoded_shares)), len(b''.join(data0s)))
self.failUnlessEqual(len(decoded_shares), len(data0s))
for (i, (x, y)) in enumerate(zip(data0s, decoded_shares)):
self.failUnlessEqual(x, y, "%s: %r != %r.... first share was %r" % (str(i), x, y, data0s[0],))
self.failUnless(b''.join(decoded_shares) == b''.join(data0s), "%s" % ("???",))
# 0data0sclipped = tuple(data0s)
# data0sclipped[-1] =
# self.failUnless(tuple(decoded_shares) == tuple(data0s))
def _decode_some(res):
log.msg("_decode_some")
# decode with a minimal subset of the shares
some_shares = self.shares[:required_shares]
some_shareids = self.shareids[:required_shares]
return _decode((some_shares, some_shareids))
d.addCallback(_decode_some)
d.addCallback(_check_data)
def _decode_some_random(res):
log.msg("_decode_some_random")
# use a randomly-selected minimal subset
l = random.sample(list(zip(self.shares, self.shareids)), required_shares)
some_shares = [ x[0] for x in l ]
some_shareids = [ x[1] for x in l ]
return _decode((some_shares, some_shareids))
d.addCallback(_decode_some_random)
d.addCallback(_check_data)
def _decode_multiple(res):
log.msg("_decode_multiple")
# make sure we can re-use the decoder object
shares1 = random.sample(self.shares, required_shares)
sharesl1 = random.sample(list(zip(self.shares, self.shareids)), required_shares)
shares1 = [ x[0] for x in sharesl1 ]
shareids1 = [ x[1] for x in sharesl1 ]
sharesl2 = random.sample(list(zip(self.shares, self.shareids)), required_shares)
shares2 = [ x[0] for x in sharesl2 ]
shareids2 = [ x[1] for x in sharesl2 ]
dec = CRSDecoder()
dec.set_params(*params)
d1 = dec.decode(shares1, shareids1)
d1.addCallback(_check_data)
d1.addCallback(lambda res: dec.decode(shares2, shareids2))
d1.addCallback(_check_data)
return d1
d.addCallback(_decode_multiple)
return d
