def test_tiny_reads(self):
# ticket #1223 points out three problems:
# repairer reads beyond end of input file
# new-downloader does not tolerate overreads
# uploader does lots of tiny reads, inefficient
self.basedir = "repairer/Repairer/test_tiny_reads"
self.set_up_grid()
c0 = self.g.clients[0]
DATA = "a" * 135
c0.encoding_params['k'] = 22
c0.encoding_params['n'] = 66
d = c0.upload(upload.Data(DATA, convergence=""))
def _then(ur):
self.uri = ur.get_uri()
self.delete_shares_numbered(self.uri, [0])
self.c0_filenode = c0.create_node_from_uri(ur.get_uri())
self._stash_counts()
return self.c0_filenode.check_and_repair(Monitor())
d.addCallback(_then)
def _check(ign):
(r, a, w) = self._get_delta_counts()
# when the uploader (driven by the repairer) does full-segment
# reads, this makes 44 server read calls (2*k). Before, when it
# was doing input_chunk_size reads (7 bytes), it was doing over
# 400.
self.failIf(r > 100, "too many reads: %d>100" % r)
d.addCallback(_check)
return d
def _set_up_tree(self):
# add immutable file at root
immutable = upload.Data("immutable file contents", None)
d = self.root.add_file(u"immutable", immutable)
# `mtime' and `linkmotime' both set
md_both = {'mtime': self.FALL_OF_BERLIN_WALL,
'tahoe': {'linkmotime': self.TURN_OF_MILLENIUM}}
d.addCallback(lambda _: self._set_metadata(u"immutable", md_both))
# add link to root from root
d.addCallback(lambda _: self.root.set_node(u"loop", self.root))
# `mtime' set, but no `linkmotime'
md_just_mtime = {'mtime': self.FALL_OF_BERLIN_WALL, 'tahoe': {}}
d.addCallback(lambda _: self._set_metadata(u"loop", md_just_mtime))
# add mutable file at root
mutable = publish.MutableData("mutable file contents")
d.addCallback(lambda _: self.client.create_mutable_file(mutable))
d.addCallback(lambda node: self.root.set_node(u"mutable", node))
# neither `mtime' nor `linkmotime' set
d.addCallback(lambda _: self._set_metadata(u"mutable", {}))
return d
def _set_up(self, mutable, testdir, num_clients=1, num_servers=10):
self.mutable = mutable
if mutable:
self.basedir = "hung_server/mutable_" + testdir
else:
self.basedir = "hung_server/immutable_" + testdir
self.set_up_grid(num_clients=num_clients, num_servers=num_servers)
self.c0 = self.g.clients[0]
nm = self.c0.nodemaker
self.servers = sorted([(s.get_serverid(), s.get_rref())
for s in nm.storage_broker.get_connected_servers()])
self.servers = self.servers[5:] + self.servers[:5]
if mutable:
uploadable = MutableData(mutable_plaintext)
d = nm.create_mutable_file(uploadable)
def _uploaded_mutable(node):
self.uri = node.get_uri()
self.shares = self.find_uri_shares(self.uri)
d.addCallback(_uploaded_mutable)
else:
data = upload.Data(immutable_plaintext, convergence="")
d = self.c0.upload(data)
def _uploaded_immutable(upload_res):
self.uri = upload_res.get_uri()
self.shares = self.find_uri_shares(self.uri)
d.addCallback(_uploaded_immutable)
return d
def do_encode(self, max_segment_size, datalen, NUM_SHARES, NUM_SEGMENTS,
expected_block_hashes, expected_share_hashes):
data = make_data(datalen)
# force use of multiple segments
e = encode.Encoder()
u = upload.Data(data, convergence="some convergence string")
u.max_segment_size = max_segment_size
u.encoding_param_k = 25
u.encoding_param_happy = 75
u.encoding_param_n = 100
eu = upload.EncryptAnUploadable(u)
d = e.set_encrypted_uploadable(eu)
all_shareholders = []
def _ready(res):
k,happy,n = e.get_param("share_counts")
_assert(n == NUM_SHARES) # else we'll be completely confused
numsegs = e.get_param("num_segments")
_assert(numsegs == NUM_SEGMENTS, numsegs, NUM_SEGMENTS)
segsize = e.get_param("segment_size")
_assert( (NUM_SEGMENTS-1)*segsize < len(data) <= NUM_SEGMENTS*segsize,
NUM_SEGMENTS, segsize,
(NUM_SEGMENTS-1)*segsize, len(data), NUM_SEGMENTS*segsize)
shareholders = {}
servermap = {}
for shnum in range(NUM_SHARES):
peer = FakeBucketReaderWriterProxy()
shareholders[shnum] = peer
servermap.setdefault(shnum, set()).add(peer.get_peerid())
all_shareholders.append(peer)
e.set_shareholders(shareholders, servermap)
return e.start()
d.addCallback(_ready)
def _check(res):
verifycap = res
self.failUnless(isinstance(verifycap.uri_extension_hash, str))
self.failUnlessEqual(len(verifycap.uri_extension_hash), 32)
for i,peer in enumerate(all_shareholders):
self.failUnless(peer.closed)
self.failUnlessEqual(len(peer.blocks), NUM_SEGMENTS)
# each peer gets a full tree of block hashes. For 3 or 4
# segments, that's 7 hashes. For 5 segments it's 15 hashes.
self.failUnlessEqual(len(peer.block_hashes),
expected_block_hashes)
for h in peer.block_hashes:
self.failUnlessEqual(len(h), 32)
# each peer also gets their necessary chain of share hashes.
# For 100 shares (rounded up to 128 leaves), that's 8 hashes
self.failUnlessEqual(len(peer.share_hashes),
expected_share_hashes)
for (hashnum, h) in peer.share_hashes:
self.failUnless(isinstance(hashnum, int))
self.failUnlessEqual(len(h), 32)
d.addCallback(_check)
return d
def upload(self, data):
u = upload.Data(data, None)
u.max_segment_size = 25
u.encoding_param_k = 25
u.encoding_param_happy = 1
u.encoding_param_n = 100
d = self.c0.upload(u)
d.addCallback(lambda ur: self.c0.create_node_from_uri(ur.uri))
# returns a FileNode
return d
def upload_and_stash(self):
c0 = self.g.clients[0]
c1 = self.g.clients[1]
c0.encoding_params['max_segment_size'] = 12
d = c0.upload(upload.Data(common.TEST_DATA, convergence=""))
def _stash_uri(ur):
self.uri = ur.get_uri()
self.c0_filenode = c0.create_node_from_uri(ur.get_uri())
self.c1_filenode = c1.create_node_from_uri(ur.get_uri())
d.addCallback(_stash_uri)
return d
def upload_and_stash(self):
c0 = self.g.clients[0]
c1 = self.g.clients[1]
c0.DEFAULT_ENCODING_PARAMETERS['max_segment_size'] = 12
d = c0.upload(upload.Data(common.TEST_DATA, convergence=""))
def _stash_uri(ur):
self.uri = ur.uri
self.c0_filenode = c0.create_node_from_uri(ur.uri)
self.c1_filenode = c1.create_node_from_uri(ur.uri)
d.addCallback(_stash_uri)
return d
def _got_rootnode(n):
# Add a few nodes.
self._dircap = n.get_uri()
nm = n._nodemaker
# The uploaders may run at the same time, so we need two
# MutableData instances or they'll fight over offsets &c and
# break.
mutable_data = MutableData(b"data" * 100000)
mutable_data2 = MutableData(b"data" * 100000)
# Add both kinds of mutable node.
d1 = nm.create_mutable_file(mutable_data, version=MDMF_VERSION)
d2 = nm.create_mutable_file(mutable_data2, version=SDMF_VERSION)
# Add an immutable node. We do this through the directory,
# with add_file.
immutable_data = upload.Data(b"immutable data" * 100000,
convergence=b"")
d3 = n.add_file(u"immutable", immutable_data)
ds = [d1, d2, d3]
dl = defer.DeferredList(ds)
def _made_files(args):
(r1, r2, r3) = args
self.failUnless(r1[0])
self.failUnless(r2[0])
self.failUnless(r3[0])
# r1, r2, and r3 contain nodes.
mdmf_node = r1[1]
sdmf_node = r2[1]
imm_node = r3[1]
self._mdmf_uri = mdmf_node.get_uri()
self._mdmf_readonly_uri = mdmf_node.get_readonly_uri()
self._sdmf_uri = mdmf_node.get_uri()
self._sdmf_readonly_uri = sdmf_node.get_readonly_uri()
self._imm_uri = imm_node.get_uri()
d1 = n.set_node(u"mdmf", mdmf_node)
d2 = n.set_node(u"sdmf", sdmf_node)
return defer.DeferredList([d1, d2])
# We can now list the directory by listing self._dircap.
dl.addCallback(_made_files)
return dl
def find_share_for_target(target):
target_s = base32.b2a(target)
prefix = "The first share of this file will be placed on " + target_s + "\n"
prefix += "This data is random: "
attempts = 0
while True:
attempts += 1
suffix = base32.b2a(os.urandom(10))
if verbose: print " trying", suffix,
data = prefix + suffix + "\n"
assert len(data) > 55 # no LIT files
# now, what storage index will this get?
u = upload.Data(data, convergence)
eu = upload.EncryptAnUploadable(u)
d = eu.get_storage_index() # this happens to run synchronously
def _got_si(si, data=data):
if verbose: print "SI", base32.b2a(si),
peerlist = get_permuted_peers(si)
if peerlist[0] == target:
# great!
if verbose: print " yay!"
fn = base32.b2a(target)
if nodes[target]:
nickname = nodes[target].replace("/", "_")
fn += "-" + nickname
fn += ".txt"
fn = os.path.join("canaries", fn)
open(fn, "w").write(data)
return True
# nope, must try again
if verbose: print " boo"
return False
d.addCallback(_got_si)
# get sneaky and look inside the Deferred for the synchronous result
if d.result:
return attempts
def _stash_root_and_create_file(n):
self.rootnode = n
self.rooturi = n.get_uri()
return n.add_file(u"g\u00F6\u00F6d",
upload.Data(DATA, convergence=""))
def test_deep_check(self):
self.basedir = "cli/Check/deep_check"
self.set_up_grid()
c0 = self.g.clients[0]
self.uris = {}
self.fileurls = {}
DATA = "data" * 100
quoted_good = quote_output(u"g\u00F6\u00F6d")
d = c0.create_dirnode()
def _stash_root_and_create_file(n):
self.rootnode = n
self.rooturi = n.get_uri()
return n.add_file(u"g\u00F6\u00F6d",
upload.Data(DATA, convergence=""))
d.addCallback(_stash_root_and_create_file)
def _stash_uri(fn, which):
self.uris[which] = fn.get_uri()
return fn
d.addCallback(_stash_uri, u"g\u00F6\u00F6d")
d.addCallback(lambda ign: self.rootnode.add_file(
u"small", upload.Data("literal", convergence="")))
d.addCallback(_stash_uri, "small")
d.addCallback(
lambda ign: c0.create_mutable_file(MutableData(DATA + "1")))
d.addCallback(lambda fn: self.rootnode.set_node(u"mutable", fn))
d.addCallback(_stash_uri, "mutable")
d.addCallback(lambda ign: self.do_cli("deep-check", self.rooturi))
def _check1(args):
(rc, out, err) = args
self.failUnlessReallyEqual(err, "")
self.failUnlessReallyEqual(rc, 0)
lines = out.splitlines()
self.failUnless(
"done: 4 objects checked, 4 healthy, 0 unhealthy" in lines,
out)
d.addCallback(_check1)
# root
# root/g\u00F6\u00F6d
# root/small
# root/mutable
d.addCallback(
lambda ign: self.do_cli("deep-check", "--verbose", self.rooturi))
def _check2(args):
(rc, out, err) = args
self.failUnlessReallyEqual(err, "")
self.failUnlessReallyEqual(rc, 0)
lines = out.splitlines()
self.failUnless("'<root>': Healthy" in lines, out)
self.failUnless("'small': Healthy (LIT)" in lines, out)
self.failUnless((quoted_good + ": Healthy") in lines, out)
self.failUnless("'mutable': Healthy" in lines, out)
self.failUnless(
"done: 4 objects checked, 4 healthy, 0 unhealthy" in lines,
out)
d.addCallback(_check2)
d.addCallback(lambda ign: self.do_cli("stats", self.rooturi))
def _check_stats(args):
(rc, out, err) = args
self.failUnlessReallyEqual(err, "")
self.failUnlessReallyEqual(rc, 0)
lines = out.splitlines()
self.failUnlessIn(" count-immutable-files: 1", lines)
self.failUnlessIn(" count-mutable-files: 1", lines)
self.failUnlessIn(" count-literal-files: 1", lines)
self.failUnlessIn(" count-directories: 1", lines)
self.failUnlessIn(" size-immutable-files: 400", lines)
self.failUnlessIn("Size Histogram:", lines)
self.failUnlessIn(" 4-10 : 1 (10 B, 10 B)", lines)
self.failUnlessIn(" 317-1000 : 1 (1000 B, 1000 B)", lines)
d.addCallback(_check_stats)
def _clobber_shares(ignored):
shares = self.find_uri_shares(self.uris[u"g\u00F6\u00F6d"])
self.failUnlessReallyEqual(len(shares), 10)
os.unlink(shares[0][2])
shares = self.find_uri_shares(self.uris["mutable"])
cso = debug.CorruptShareOptions()
cso.stdout = StringIO()
cso.parseOptions([shares[1][2]])
storage_index = uri.from_string(
self.uris["mutable"]).get_storage_index()
self._corrupt_share_line = " corrupt: server %s, SI %s, shnum %d" % \
(base32.b2a(shares[1][1]),
base32.b2a(storage_index),
shares[1][0])
debug.corrupt_share(cso)
d.addCallback(_clobber_shares)
# root
# root/g\u00F6\u00F6d [9 shares]
# root/small
# root/mutable [1 corrupt share]
d.addCallback(
lambda ign: self.do_cli("deep-check", "--verbose", self.rooturi))
def _check3(args):
(rc, out, err) = args
self.failUnlessReallyEqual(err, "")
self.failUnlessReallyEqual(rc, 0)
lines = out.splitlines()
self.failUnless("'<root>': Healthy" in lines, out)
self.failUnless("'small': Healthy (LIT)" in lines, out)
self.failUnless("'mutable': Healthy" in lines,
out) # needs verifier
self.failUnless(
(quoted_good + ": Not Healthy: 9 shares (enc 3-of-10)")
in lines, out)
self.failIf(self._corrupt_share_line in lines, out)
self.failUnless(
"done: 4 objects checked, 3 healthy, 1 unhealthy" in lines,
out)
d.addCallback(_check3)
d.addCallback(lambda ign: self.do_cli("deep-check", "--verbose",
"--verify", self.rooturi))
def _check4(args):
(rc, out, err) = args
self.failUnlessReallyEqual(err, "")
self.failUnlessReallyEqual(rc, 0)
lines = out.splitlines()
self.failUnless("'<root>': Healthy" in lines, out)
self.failUnless("'small': Healthy (LIT)" in lines, out)
mutable = [l for l in lines if l.startswith("'mutable'")][0]
self.failUnless(
mutable.startswith(
"'mutable': Unhealthy: 9 shares (enc 3-of-10)"), mutable)
self.failUnless(self._corrupt_share_line in lines, out)
self.failUnless(
(quoted_good + ": Not Healthy: 9 shares (enc 3-of-10)")
in lines, out)
self.failUnless(
"done: 4 objects checked, 2 healthy, 2 unhealthy" in lines,
out)
d.addCallback(_check4)
d.addCallback(
lambda ign: self.do_cli("deep-check", "--raw", self.rooturi))
def _check5(args):
(rc, out, err) = args
self.failUnlessReallyEqual(err, "")
self.failUnlessReallyEqual(rc, 0)
lines = out.splitlines()
units = [json.loads(line) for line in lines]
# root, small, g\u00F6\u00F6d, mutable, stats
self.failUnlessReallyEqual(len(units), 4 + 1)
d.addCallback(_check5)
d.addCallback(lambda ign: self.do_cli(
"deep-check", "--verbose", "--verify", "--repair", self.rooturi))
def _check6(args):
(rc, out, err) = args
self.failUnlessReallyEqual(err, "")
self.failUnlessReallyEqual(rc, 0)
lines = out.splitlines()
self.failUnless("'<root>': healthy" in lines, out)
self.failUnless("'small': healthy" in lines, out)
self.failUnless("'mutable': not healthy" in lines, out)
self.failUnless(self._corrupt_share_line in lines, out)
self.failUnless((quoted_good + ": not healthy") in lines, out)
self.failUnless("done: 4 objects checked" in lines, out)
self.failUnless(" pre-repair: 2 healthy, 2 unhealthy" in lines,
out)
self.failUnless(
" 2 repairs attempted, 2 successful, 0 failed" in lines, out)
self.failUnless(" post-repair: 4 healthy, 0 unhealthy" in lines,
out)
d.addCallback(_check6)
# now add a subdir, and a file below that, then make the subdir
# unrecoverable
d.addCallback(lambda ign: self.rootnode.create_subdirectory(u"subdir"))
d.addCallback(_stash_uri, "subdir")
d.addCallback(
lambda fn: fn.add_file(u"subfile", upload.Data(DATA + "2", "")))
d.addCallback(lambda ign: self.delete_shares_numbered(
self.uris["subdir"], range(10)))
# root
# rootg\u00F6\u00F6d/
# root/small
# root/mutable
# root/subdir [unrecoverable: 0 shares]
# root/subfile
d.addCallback(lambda ign: self.do_cli("manifest", self.rooturi))
def _manifest_failed(args):
(rc, out, err) = args
self.failIfEqual(rc, 0)
self.failUnlessIn("ERROR: UnrecoverableFileError", err)
# the fatal directory should still show up, as the last line
self.failUnlessIn(" subdir\n", out)
d.addCallback(_manifest_failed)
d.addCallback(lambda ign: self.do_cli("deep-check", self.rooturi))
def _deep_check_failed(args):
(rc, out, err) = args
self.failIfEqual(rc, 0)
self.failUnlessIn("ERROR: UnrecoverableFileError", err)
# we want to make sure that the error indication is the last
# thing that gets emitted
self.failIf("done:" in out, out)
d.addCallback(_deep_check_failed)
# this test is disabled until the deep-repair response to an
# unrepairable directory is fixed. The failure-to-repair should not
# throw an exception, but the failure-to-traverse that follows
# should throw UnrecoverableFileError.
#d.addCallback(lambda ign:
# self.do_cli("deep-check", "--repair", self.rooturi))
#def _deep_check_repair_failed((rc, out, err)):
# self.failIfEqual(rc, 0)
# print err
# self.failUnlessIn("ERROR: UnrecoverableFileError", err)
# self.failIf("done:" in out, out)
#d.addCallback(_deep_check_repair_failed)
return d
def test_check(self):
self.basedir = "cli/Check/check"
self.set_up_grid()
c0 = self.g.clients[0]
DATA = "data" * 100
DATA_uploadable = MutableData(DATA)
d = c0.create_mutable_file(DATA_uploadable)
def _stash_uri(n):
self.uri = n.get_uri()
d.addCallback(_stash_uri)
d.addCallback(lambda ign: self.do_cli("check", self.uri))
def _check1(args):
(rc, out, err) = args
self.failUnlessReallyEqual(err, "")
self.failUnlessReallyEqual(rc, 0)
lines = out.splitlines()
self.failUnless("Summary: Healthy" in lines, out)
self.failUnless(" good-shares: 10 (encoding is 3-of-10)" in lines,
out)
d.addCallback(_check1)
d.addCallback(lambda ign: self.do_cli("check", "--raw", self.uri))
def _check2(args):
(rc, out, err) = args
self.failUnlessReallyEqual(err, "")
self.failUnlessReallyEqual(rc, 0)
data = json.loads(out)
self.failUnlessReallyEqual(to_bytes(data["summary"]), "Healthy")
self.failUnlessReallyEqual(data["results"]["healthy"], True)
d.addCallback(_check2)
d.addCallback(
lambda ign: c0.upload(upload.Data("literal", convergence="")))
def _stash_lit_uri(n):
self.lit_uri = n.get_uri()
d.addCallback(_stash_lit_uri)
d.addCallback(lambda ign: self.do_cli("check", self.lit_uri))
def _check_lit(args):
(rc, out, err) = args
self.failUnlessReallyEqual(err, "")
self.failUnlessReallyEqual(rc, 0)
lines = out.splitlines()
self.failUnless("Summary: Healthy (LIT)" in lines, out)
d.addCallback(_check_lit)
d.addCallback(lambda ign: self.do_cli("check", "--raw", self.lit_uri))
def _check_lit_raw(args):
(rc, out, err) = args
self.failUnlessReallyEqual(err, "")
self.failUnlessReallyEqual(rc, 0)
data = json.loads(out)
self.failUnlessReallyEqual(data["results"]["healthy"], True)
d.addCallback(_check_lit_raw)
d.addCallback(
lambda ign: c0.create_immutable_dirnode({}, convergence=""))
def _stash_lit_dir_uri(n):
self.lit_dir_uri = n.get_uri()
d.addCallback(_stash_lit_dir_uri)
d.addCallback(lambda ign: self.do_cli("check", self.lit_dir_uri))
d.addCallback(_check_lit)
d.addCallback(lambda ign: self.do_cli("check", "--raw", self.lit_uri))
d.addCallback(_check_lit_raw)
def _clobber_shares(ignored):
# delete one, corrupt a second
shares = self.find_uri_shares(self.uri)
self.failUnlessReallyEqual(len(shares), 10)
os.unlink(shares[0][2])
cso = debug.CorruptShareOptions()
cso.stdout = StringIO()
cso.parseOptions([shares[1][2]])
storage_index = uri.from_string(self.uri).get_storage_index()
self._corrupt_share_line = " server %s, SI %s, shnum %d" % \
(base32.b2a(shares[1][1]),
base32.b2a(storage_index),
shares[1][0])
debug.corrupt_share(cso)
d.addCallback(_clobber_shares)
d.addCallback(lambda ign: self.do_cli("check", "--verify", self.uri))
def _check3(args):
(rc, out, err) = args
self.failUnlessReallyEqual(err, "")
self.failUnlessReallyEqual(rc, 0)
lines = out.splitlines()
summary = [l for l in lines if l.startswith("Summary")][0]
self.failUnless(
"Summary: Unhealthy: 8 shares (enc 3-of-10)" in summary,
summary)
self.failUnless(" good-shares: 8 (encoding is 3-of-10)" in lines,
out)
self.failUnless(" corrupt shares:" in lines, out)
self.failUnless(self._corrupt_share_line in lines, out)
d.addCallback(_check3)
d.addCallback(
lambda ign: self.do_cli("check", "--verify", "--raw", self.uri))
def _check3_raw(args):
(rc, out, err) = args
self.failUnlessReallyEqual(err, "")
self.failUnlessReallyEqual(rc, 0)
data = json.loads(out)
self.failUnlessReallyEqual(data["results"]["healthy"], False)
self.failUnlessIn("Unhealthy: 8 shares (enc 3-of-10)",
data["summary"])
self.failUnlessReallyEqual(data["results"]["count-shares-good"], 8)
self.failUnlessReallyEqual(data["results"]["count-corrupt-shares"],
1)
self.failUnlessIn("list-corrupt-shares", data["results"])
d.addCallback(_check3_raw)
d.addCallback(
lambda ign: self.do_cli("check", "--verify", "--repair", self.uri))
def _check4(args):
(rc, out, err) = args
self.failUnlessReallyEqual(err, "")
self.failUnlessReallyEqual(rc, 0)
lines = out.splitlines()
self.failUnless("Summary: not healthy" in lines, out)
self.failUnless(" good-shares: 8 (encoding is 3-of-10)" in lines,
out)
self.failUnless(" corrupt shares:" in lines, out)
self.failUnless(self._corrupt_share_line in lines, out)
self.failUnless(" repair successful" in lines, out)
d.addCallback(_check4)
d.addCallback(
lambda ign: self.do_cli("check", "--verify", "--repair", self.uri))
def _check5(args):
(rc, out, err) = args
self.failUnlessReallyEqual(err, "")
self.failUnlessReallyEqual(rc, 0)
lines = out.splitlines()
self.failUnless("Summary: healthy" in lines, out)
self.failUnless(" good-shares: 10 (encoding is 3-of-10)" in lines,
out)
self.failIf(" corrupt shares:" in lines, out)
d.addCallback(_check5)
return d
def upload_data(uploader, data, convergence):
u = upload.Data(data, convergence=convergence)
return uploader.upload(u)
def _stash_root_and_create_file(n):
self.rootnode = n
self.rooturi = str(n.get_uri(), "utf-8")
return n.add_file(u"g\u00F6\u00F6d",
upload.Data(small, convergence=b""))
