def test_tiny_reads(self): # ticket #1223 points out three problems: # repairer reads beyond end of input file # new-downloader does not tolerate overreads # uploader does lots of tiny reads, inefficient self.basedir = "repairer/Repairer/test_tiny_reads" self.set_up_grid() c0 = self.g.clients[0] DATA = "a" * 135 c0.encoding_params['k'] = 22 c0.encoding_params['n'] = 66 d = c0.upload(upload.Data(DATA, convergence="")) def _then(ur): self.uri = ur.get_uri() self.delete_shares_numbered(self.uri, [0]) self.c0_filenode = c0.create_node_from_uri(ur.get_uri()) self._stash_counts() return self.c0_filenode.check_and_repair(Monitor()) d.addCallback(_then) def _check(ign): (r, a, w) = self._get_delta_counts() # when the uploader (driven by the repairer) does full-segment # reads, this makes 44 server read calls (2*k). Before, when it # was doing input_chunk_size reads (7 bytes), it was doing over # 400. self.failIf(r > 100, "too many reads: %d>100" % r) d.addCallback(_check) return d
def _set_up_tree(self): # add immutable file at root immutable = upload.Data("immutable file contents", None) d = self.root.add_file(u"immutable", immutable) # `mtime' and `linkmotime' both set md_both = {'mtime': self.FALL_OF_BERLIN_WALL, 'tahoe': {'linkmotime': self.TURN_OF_MILLENIUM}} d.addCallback(lambda _: self._set_metadata(u"immutable", md_both)) # add link to root from root d.addCallback(lambda _: self.root.set_node(u"loop", self.root)) # `mtime' set, but no `linkmotime' md_just_mtime = {'mtime': self.FALL_OF_BERLIN_WALL, 'tahoe': {}} d.addCallback(lambda _: self._set_metadata(u"loop", md_just_mtime)) # add mutable file at root mutable = publish.MutableData("mutable file contents") d.addCallback(lambda _: self.client.create_mutable_file(mutable)) d.addCallback(lambda node: self.root.set_node(u"mutable", node)) # neither `mtime' nor `linkmotime' set d.addCallback(lambda _: self._set_metadata(u"mutable", {})) return d
def _set_up(self, mutable, testdir, num_clients=1, num_servers=10): self.mutable = mutable if mutable: self.basedir = "hung_server/mutable_" + testdir else: self.basedir = "hung_server/immutable_" + testdir self.set_up_grid(num_clients=num_clients, num_servers=num_servers) self.c0 = self.g.clients[0] nm = self.c0.nodemaker self.servers = sorted([(s.get_serverid(), s.get_rref()) for s in nm.storage_broker.get_connected_servers()]) self.servers = self.servers[5:] + self.servers[:5] if mutable: uploadable = MutableData(mutable_plaintext) d = nm.create_mutable_file(uploadable) def _uploaded_mutable(node): self.uri = node.get_uri() self.shares = self.find_uri_shares(self.uri) d.addCallback(_uploaded_mutable) else: data = upload.Data(immutable_plaintext, convergence="") d = self.c0.upload(data) def _uploaded_immutable(upload_res): self.uri = upload_res.get_uri() self.shares = self.find_uri_shares(self.uri) d.addCallback(_uploaded_immutable) return d
def do_encode(self, max_segment_size, datalen, NUM_SHARES, NUM_SEGMENTS, expected_block_hashes, expected_share_hashes): data = make_data(datalen) # force use of multiple segments e = encode.Encoder() u = upload.Data(data, convergence="some convergence string") u.max_segment_size = max_segment_size u.encoding_param_k = 25 u.encoding_param_happy = 75 u.encoding_param_n = 100 eu = upload.EncryptAnUploadable(u) d = e.set_encrypted_uploadable(eu) all_shareholders = [] def _ready(res): k,happy,n = e.get_param("share_counts") _assert(n == NUM_SHARES) # else we'll be completely confused numsegs = e.get_param("num_segments") _assert(numsegs == NUM_SEGMENTS, numsegs, NUM_SEGMENTS) segsize = e.get_param("segment_size") _assert( (NUM_SEGMENTS-1)*segsize < len(data) <= NUM_SEGMENTS*segsize, NUM_SEGMENTS, segsize, (NUM_SEGMENTS-1)*segsize, len(data), NUM_SEGMENTS*segsize) shareholders = {} servermap = {} for shnum in range(NUM_SHARES): peer = FakeBucketReaderWriterProxy() shareholders[shnum] = peer servermap.setdefault(shnum, set()).add(peer.get_peerid()) all_shareholders.append(peer) e.set_shareholders(shareholders, servermap) return e.start() d.addCallback(_ready) def _check(res): verifycap = res self.failUnless(isinstance(verifycap.uri_extension_hash, str)) self.failUnlessEqual(len(verifycap.uri_extension_hash), 32) for i,peer in enumerate(all_shareholders): self.failUnless(peer.closed) self.failUnlessEqual(len(peer.blocks), NUM_SEGMENTS) # each peer gets a full tree of block hashes. For 3 or 4 # segments, that's 7 hashes. For 5 segments it's 15 hashes. self.failUnlessEqual(len(peer.block_hashes), expected_block_hashes) for h in peer.block_hashes: self.failUnlessEqual(len(h), 32) # each peer also gets their necessary chain of share hashes. # For 100 shares (rounded up to 128 leaves), that's 8 hashes self.failUnlessEqual(len(peer.share_hashes), expected_share_hashes) for (hashnum, h) in peer.share_hashes: self.failUnless(isinstance(hashnum, int)) self.failUnlessEqual(len(h), 32) d.addCallback(_check) return d
def upload(self, data): u = upload.Data(data, None) u.max_segment_size = 25 u.encoding_param_k = 25 u.encoding_param_happy = 1 u.encoding_param_n = 100 d = self.c0.upload(u) d.addCallback(lambda ur: self.c0.create_node_from_uri(ur.uri)) # returns a FileNode return d
def upload_and_stash(self): c0 = self.g.clients[0] c1 = self.g.clients[1] c0.encoding_params['max_segment_size'] = 12 d = c0.upload(upload.Data(common.TEST_DATA, convergence="")) def _stash_uri(ur): self.uri = ur.get_uri() self.c0_filenode = c0.create_node_from_uri(ur.get_uri()) self.c1_filenode = c1.create_node_from_uri(ur.get_uri()) d.addCallback(_stash_uri) return d
def upload_and_stash(self): c0 = self.g.clients[0] c1 = self.g.clients[1] c0.DEFAULT_ENCODING_PARAMETERS['max_segment_size'] = 12 d = c0.upload(upload.Data(common.TEST_DATA, convergence="")) def _stash_uri(ur): self.uri = ur.uri self.c0_filenode = c0.create_node_from_uri(ur.uri) self.c1_filenode = c1.create_node_from_uri(ur.uri) d.addCallback(_stash_uri) return d
def _got_rootnode(n): # Add a few nodes. self._dircap = n.get_uri() nm = n._nodemaker # The uploaders may run at the same time, so we need two # MutableData instances or they'll fight over offsets &c and # break. mutable_data = MutableData(b"data" * 100000) mutable_data2 = MutableData(b"data" * 100000) # Add both kinds of mutable node. d1 = nm.create_mutable_file(mutable_data, version=MDMF_VERSION) d2 = nm.create_mutable_file(mutable_data2, version=SDMF_VERSION) # Add an immutable node. We do this through the directory, # with add_file. immutable_data = upload.Data(b"immutable data" * 100000, convergence=b"") d3 = n.add_file(u"immutable", immutable_data) ds = [d1, d2, d3] dl = defer.DeferredList(ds) def _made_files(args): (r1, r2, r3) = args self.failUnless(r1[0]) self.failUnless(r2[0]) self.failUnless(r3[0]) # r1, r2, and r3 contain nodes. mdmf_node = r1[1] sdmf_node = r2[1] imm_node = r3[1] self._mdmf_uri = mdmf_node.get_uri() self._mdmf_readonly_uri = mdmf_node.get_readonly_uri() self._sdmf_uri = mdmf_node.get_uri() self._sdmf_readonly_uri = sdmf_node.get_readonly_uri() self._imm_uri = imm_node.get_uri() d1 = n.set_node(u"mdmf", mdmf_node) d2 = n.set_node(u"sdmf", sdmf_node) return defer.DeferredList([d1, d2]) # We can now list the directory by listing self._dircap. dl.addCallback(_made_files) return dl
def find_share_for_target(target): target_s = base32.b2a(target) prefix = "The first share of this file will be placed on " + target_s + "\n" prefix += "This data is random: " attempts = 0 while True: attempts += 1 suffix = base32.b2a(os.urandom(10)) if verbose: print " trying", suffix, data = prefix + suffix + "\n" assert len(data) > 55 # no LIT files # now, what storage index will this get? u = upload.Data(data, convergence) eu = upload.EncryptAnUploadable(u) d = eu.get_storage_index() # this happens to run synchronously def _got_si(si, data=data): if verbose: print "SI", base32.b2a(si), peerlist = get_permuted_peers(si) if peerlist[0] == target: # great! if verbose: print " yay!" fn = base32.b2a(target) if nodes[target]: nickname = nodes[target].replace("/", "_") fn += "-" + nickname fn += ".txt" fn = os.path.join("canaries", fn) open(fn, "w").write(data) return True # nope, must try again if verbose: print " boo" return False d.addCallback(_got_si) # get sneaky and look inside the Deferred for the synchronous result if d.result: return attempts
def _stash_root_and_create_file(n): self.rootnode = n self.rooturi = n.get_uri() return n.add_file(u"g\u00F6\u00F6d", upload.Data(DATA, convergence=""))
def test_deep_check(self): self.basedir = "cli/Check/deep_check" self.set_up_grid() c0 = self.g.clients[0] self.uris = {} self.fileurls = {} DATA = "data" * 100 quoted_good = quote_output(u"g\u00F6\u00F6d") d = c0.create_dirnode() def _stash_root_and_create_file(n): self.rootnode = n self.rooturi = n.get_uri() return n.add_file(u"g\u00F6\u00F6d", upload.Data(DATA, convergence="")) d.addCallback(_stash_root_and_create_file) def _stash_uri(fn, which): self.uris[which] = fn.get_uri() return fn d.addCallback(_stash_uri, u"g\u00F6\u00F6d") d.addCallback(lambda ign: self.rootnode.add_file( u"small", upload.Data("literal", convergence=""))) d.addCallback(_stash_uri, "small") d.addCallback( lambda ign: c0.create_mutable_file(MutableData(DATA + "1"))) d.addCallback(lambda fn: self.rootnode.set_node(u"mutable", fn)) d.addCallback(_stash_uri, "mutable") d.addCallback(lambda ign: self.do_cli("deep-check", self.rooturi)) def _check1(args): (rc, out, err) = args self.failUnlessReallyEqual(err, "") self.failUnlessReallyEqual(rc, 0) lines = out.splitlines() self.failUnless( "done: 4 objects checked, 4 healthy, 0 unhealthy" in lines, out) d.addCallback(_check1) # root # root/g\u00F6\u00F6d # root/small # root/mutable d.addCallback( lambda ign: self.do_cli("deep-check", "--verbose", self.rooturi)) def _check2(args): (rc, out, err) = args self.failUnlessReallyEqual(err, "") self.failUnlessReallyEqual(rc, 0) lines = out.splitlines() self.failUnless("'<root>': Healthy" in lines, out) self.failUnless("'small': Healthy (LIT)" in lines, out) self.failUnless((quoted_good + ": Healthy") in lines, out) self.failUnless("'mutable': Healthy" in lines, out) self.failUnless( "done: 4 objects checked, 4 healthy, 0 unhealthy" in lines, out) d.addCallback(_check2) d.addCallback(lambda ign: self.do_cli("stats", self.rooturi)) def _check_stats(args): (rc, out, err) = args self.failUnlessReallyEqual(err, "") self.failUnlessReallyEqual(rc, 0) lines = out.splitlines() self.failUnlessIn(" count-immutable-files: 1", lines) self.failUnlessIn(" count-mutable-files: 1", lines) self.failUnlessIn(" count-literal-files: 1", lines) self.failUnlessIn(" count-directories: 1", lines) self.failUnlessIn(" size-immutable-files: 400", lines) self.failUnlessIn("Size Histogram:", lines) self.failUnlessIn(" 4-10 : 1 (10 B, 10 B)", lines) self.failUnlessIn(" 317-1000 : 1 (1000 B, 1000 B)", lines) d.addCallback(_check_stats) def _clobber_shares(ignored): shares = self.find_uri_shares(self.uris[u"g\u00F6\u00F6d"]) self.failUnlessReallyEqual(len(shares), 10) os.unlink(shares[0][2]) shares = self.find_uri_shares(self.uris["mutable"]) cso = debug.CorruptShareOptions() cso.stdout = StringIO() cso.parseOptions([shares[1][2]]) storage_index = uri.from_string( self.uris["mutable"]).get_storage_index() self._corrupt_share_line = " corrupt: server %s, SI %s, shnum %d" % \ (base32.b2a(shares[1][1]), base32.b2a(storage_index), shares[1][0]) debug.corrupt_share(cso) d.addCallback(_clobber_shares) # root # root/g\u00F6\u00F6d [9 shares] # root/small # root/mutable [1 corrupt share] d.addCallback( lambda ign: self.do_cli("deep-check", "--verbose", self.rooturi)) def _check3(args): (rc, out, err) = args self.failUnlessReallyEqual(err, "") self.failUnlessReallyEqual(rc, 0) lines = out.splitlines() self.failUnless("'<root>': Healthy" in lines, out) self.failUnless("'small': Healthy (LIT)" in lines, out) self.failUnless("'mutable': Healthy" in lines, out) # needs verifier self.failUnless( (quoted_good + ": Not Healthy: 9 shares (enc 3-of-10)") in lines, out) self.failIf(self._corrupt_share_line in lines, out) self.failUnless( "done: 4 objects checked, 3 healthy, 1 unhealthy" in lines, out) d.addCallback(_check3) d.addCallback(lambda ign: self.do_cli("deep-check", "--verbose", "--verify", self.rooturi)) def _check4(args): (rc, out, err) = args self.failUnlessReallyEqual(err, "") self.failUnlessReallyEqual(rc, 0) lines = out.splitlines() self.failUnless("'<root>': Healthy" in lines, out) self.failUnless("'small': Healthy (LIT)" in lines, out) mutable = [l for l in lines if l.startswith("'mutable'")][0] self.failUnless( mutable.startswith( "'mutable': Unhealthy: 9 shares (enc 3-of-10)"), mutable) self.failUnless(self._corrupt_share_line in lines, out) self.failUnless( (quoted_good + ": Not Healthy: 9 shares (enc 3-of-10)") in lines, out) self.failUnless( "done: 4 objects checked, 2 healthy, 2 unhealthy" in lines, out) d.addCallback(_check4) d.addCallback( lambda ign: self.do_cli("deep-check", "--raw", self.rooturi)) def _check5(args): (rc, out, err) = args self.failUnlessReallyEqual(err, "") self.failUnlessReallyEqual(rc, 0) lines = out.splitlines() units = [json.loads(line) for line in lines] # root, small, g\u00F6\u00F6d, mutable, stats self.failUnlessReallyEqual(len(units), 4 + 1) d.addCallback(_check5) d.addCallback(lambda ign: self.do_cli( "deep-check", "--verbose", "--verify", "--repair", self.rooturi)) def _check6(args): (rc, out, err) = args self.failUnlessReallyEqual(err, "") self.failUnlessReallyEqual(rc, 0) lines = out.splitlines() self.failUnless("'<root>': healthy" in lines, out) self.failUnless("'small': healthy" in lines, out) self.failUnless("'mutable': not healthy" in lines, out) self.failUnless(self._corrupt_share_line in lines, out) self.failUnless((quoted_good + ": not healthy") in lines, out) self.failUnless("done: 4 objects checked" in lines, out) self.failUnless(" pre-repair: 2 healthy, 2 unhealthy" in lines, out) self.failUnless( " 2 repairs attempted, 2 successful, 0 failed" in lines, out) self.failUnless(" post-repair: 4 healthy, 0 unhealthy" in lines, out) d.addCallback(_check6) # now add a subdir, and a file below that, then make the subdir # unrecoverable d.addCallback(lambda ign: self.rootnode.create_subdirectory(u"subdir")) d.addCallback(_stash_uri, "subdir") d.addCallback( lambda fn: fn.add_file(u"subfile", upload.Data(DATA + "2", ""))) d.addCallback(lambda ign: self.delete_shares_numbered( self.uris["subdir"], range(10))) # root # rootg\u00F6\u00F6d/ # root/small # root/mutable # root/subdir [unrecoverable: 0 shares] # root/subfile d.addCallback(lambda ign: self.do_cli("manifest", self.rooturi)) def _manifest_failed(args): (rc, out, err) = args self.failIfEqual(rc, 0) self.failUnlessIn("ERROR: UnrecoverableFileError", err) # the fatal directory should still show up, as the last line self.failUnlessIn(" subdir\n", out) d.addCallback(_manifest_failed) d.addCallback(lambda ign: self.do_cli("deep-check", self.rooturi)) def _deep_check_failed(args): (rc, out, err) = args self.failIfEqual(rc, 0) self.failUnlessIn("ERROR: UnrecoverableFileError", err) # we want to make sure that the error indication is the last # thing that gets emitted self.failIf("done:" in out, out) d.addCallback(_deep_check_failed) # this test is disabled until the deep-repair response to an # unrepairable directory is fixed. The failure-to-repair should not # throw an exception, but the failure-to-traverse that follows # should throw UnrecoverableFileError. #d.addCallback(lambda ign: # self.do_cli("deep-check", "--repair", self.rooturi)) #def _deep_check_repair_failed((rc, out, err)): # self.failIfEqual(rc, 0) # print err # self.failUnlessIn("ERROR: UnrecoverableFileError", err) # self.failIf("done:" in out, out) #d.addCallback(_deep_check_repair_failed) return d
def test_check(self): self.basedir = "cli/Check/check" self.set_up_grid() c0 = self.g.clients[0] DATA = "data" * 100 DATA_uploadable = MutableData(DATA) d = c0.create_mutable_file(DATA_uploadable) def _stash_uri(n): self.uri = n.get_uri() d.addCallback(_stash_uri) d.addCallback(lambda ign: self.do_cli("check", self.uri)) def _check1(args): (rc, out, err) = args self.failUnlessReallyEqual(err, "") self.failUnlessReallyEqual(rc, 0) lines = out.splitlines() self.failUnless("Summary: Healthy" in lines, out) self.failUnless(" good-shares: 10 (encoding is 3-of-10)" in lines, out) d.addCallback(_check1) d.addCallback(lambda ign: self.do_cli("check", "--raw", self.uri)) def _check2(args): (rc, out, err) = args self.failUnlessReallyEqual(err, "") self.failUnlessReallyEqual(rc, 0) data = json.loads(out) self.failUnlessReallyEqual(to_bytes(data["summary"]), "Healthy") self.failUnlessReallyEqual(data["results"]["healthy"], True) d.addCallback(_check2) d.addCallback( lambda ign: c0.upload(upload.Data("literal", convergence=""))) def _stash_lit_uri(n): self.lit_uri = n.get_uri() d.addCallback(_stash_lit_uri) d.addCallback(lambda ign: self.do_cli("check", self.lit_uri)) def _check_lit(args): (rc, out, err) = args self.failUnlessReallyEqual(err, "") self.failUnlessReallyEqual(rc, 0) lines = out.splitlines() self.failUnless("Summary: Healthy (LIT)" in lines, out) d.addCallback(_check_lit) d.addCallback(lambda ign: self.do_cli("check", "--raw", self.lit_uri)) def _check_lit_raw(args): (rc, out, err) = args self.failUnlessReallyEqual(err, "") self.failUnlessReallyEqual(rc, 0) data = json.loads(out) self.failUnlessReallyEqual(data["results"]["healthy"], True) d.addCallback(_check_lit_raw) d.addCallback( lambda ign: c0.create_immutable_dirnode({}, convergence="")) def _stash_lit_dir_uri(n): self.lit_dir_uri = n.get_uri() d.addCallback(_stash_lit_dir_uri) d.addCallback(lambda ign: self.do_cli("check", self.lit_dir_uri)) d.addCallback(_check_lit) d.addCallback(lambda ign: self.do_cli("check", "--raw", self.lit_uri)) d.addCallback(_check_lit_raw) def _clobber_shares(ignored): # delete one, corrupt a second shares = self.find_uri_shares(self.uri) self.failUnlessReallyEqual(len(shares), 10) os.unlink(shares[0][2]) cso = debug.CorruptShareOptions() cso.stdout = StringIO() cso.parseOptions([shares[1][2]]) storage_index = uri.from_string(self.uri).get_storage_index() self._corrupt_share_line = " server %s, SI %s, shnum %d" % \ (base32.b2a(shares[1][1]), base32.b2a(storage_index), shares[1][0]) debug.corrupt_share(cso) d.addCallback(_clobber_shares) d.addCallback(lambda ign: self.do_cli("check", "--verify", self.uri)) def _check3(args): (rc, out, err) = args self.failUnlessReallyEqual(err, "") self.failUnlessReallyEqual(rc, 0) lines = out.splitlines() summary = [l for l in lines if l.startswith("Summary")][0] self.failUnless( "Summary: Unhealthy: 8 shares (enc 3-of-10)" in summary, summary) self.failUnless(" good-shares: 8 (encoding is 3-of-10)" in lines, out) self.failUnless(" corrupt shares:" in lines, out) self.failUnless(self._corrupt_share_line in lines, out) d.addCallback(_check3) d.addCallback( lambda ign: self.do_cli("check", "--verify", "--raw", self.uri)) def _check3_raw(args): (rc, out, err) = args self.failUnlessReallyEqual(err, "") self.failUnlessReallyEqual(rc, 0) data = json.loads(out) self.failUnlessReallyEqual(data["results"]["healthy"], False) self.failUnlessIn("Unhealthy: 8 shares (enc 3-of-10)", data["summary"]) self.failUnlessReallyEqual(data["results"]["count-shares-good"], 8) self.failUnlessReallyEqual(data["results"]["count-corrupt-shares"], 1) self.failUnlessIn("list-corrupt-shares", data["results"]) d.addCallback(_check3_raw) d.addCallback( lambda ign: self.do_cli("check", "--verify", "--repair", self.uri)) def _check4(args): (rc, out, err) = args self.failUnlessReallyEqual(err, "") self.failUnlessReallyEqual(rc, 0) lines = out.splitlines() self.failUnless("Summary: not healthy" in lines, out) self.failUnless(" good-shares: 8 (encoding is 3-of-10)" in lines, out) self.failUnless(" corrupt shares:" in lines, out) self.failUnless(self._corrupt_share_line in lines, out) self.failUnless(" repair successful" in lines, out) d.addCallback(_check4) d.addCallback( lambda ign: self.do_cli("check", "--verify", "--repair", self.uri)) def _check5(args): (rc, out, err) = args self.failUnlessReallyEqual(err, "") self.failUnlessReallyEqual(rc, 0) lines = out.splitlines() self.failUnless("Summary: healthy" in lines, out) self.failUnless(" good-shares: 10 (encoding is 3-of-10)" in lines, out) self.failIf(" corrupt shares:" in lines, out) d.addCallback(_check5) return d
def upload_data(uploader, data, convergence): u = upload.Data(data, convergence=convergence) return uploader.upload(u)
def _stash_root_and_create_file(n): self.rootnode = n self.rooturi = str(n.get_uri(), "utf-8") return n.add_file(u"g\u00F6\u00F6d", upload.Data(small, convergence=b""))