def __init__(self, peer_getter, storage_index, logparent=None): self._peer_getter = peer_getter self._found_shares = set() self._storage_index = storage_index self._sharemap = dictutil.DictOfSets() self._readers = set() self._ueb_hash = None self._ueb_data = None self._logparent = logparent
def test_dict_of_sets(self): ds = dictutil.DictOfSets() ds.add(1, "a") ds.add(2, "b") ds.add(2, "b") ds.add(2, "c") self.failUnlessEqual(ds[1], set(["a"])) self.failUnlessEqual(ds[2], set(["b", "c"])) ds.discard(3, "d") # should not raise an exception ds.discard(2, "b") self.failUnlessEqual(ds[2], set(["c"])) ds.discard(2, "c") self.failIf(2 in ds) ds.add(3, "f") ds2 = dictutil.DictOfSets() ds2.add(3, "f") ds2.add(3, "g") ds2.add(4, "h") ds.update(ds2) self.failUnlessEqual(ds[1], set(["a"])) self.failUnlessEqual(ds[3], set(["f", "g"])) self.failUnlessEqual(ds[4], set(["h"]))
def test_already_uploaded(self): """ If enough shares to satisfy the needed parameter already exist, the upload succeeds without pushing any shares. """ params = FakeClient.DEFAULT_ENCODING_PARAMETERS chk_checker = partial( FakeCHKCheckerAndUEBFetcher, sharemap=dictutil.DictOfSets({ 0: {b"server0"}, 1: {b"server1"}, }), ueb_data={ "size": len(DATA), "segment_size": min(params["max_segment_size"], len(DATA)), "needed_shares": params["k"], "total_shares": params["n"], }, ) self.basedir = "helper/AssistedUpload/test_already_uploaded" self.setUpHelper( self.basedir, chk_checker=chk_checker, ) u = make_uploader(self.helper_furl, self.s) yield wait_a_few_turns() assert u._helper results = yield upload_data(u, DATA, convergence=b"some convergence string") the_uri = results.get_uri() assert b"CHK" in the_uri files = os.listdir(os.path.join(self.basedir, "CHK_encoding")) self.failUnlessEqual(files, []) files = os.listdir(os.path.join(self.basedir, "CHK_incoming")) self.failUnlessEqual(files, []) self.assertEqual( results.get_pushed_shares(), 0, )
def _make_checker_results(self, smap): self._monitor.raise_if_cancelled() healthy = True report = [] summary = [] vmap = smap.make_versionmap() recoverable = smap.recoverable_versions() unrecoverable = smap.unrecoverable_versions() if recoverable: report.append("Recoverable Versions: " + "/".join(["%d*%s" % (len(vmap[v]), smap.summarize_version(v)) for v in recoverable])) if unrecoverable: report.append("Unrecoverable Versions: " + "/".join(["%d*%s" % (len(vmap[v]), smap.summarize_version(v)) for v in unrecoverable])) if smap.unrecoverable_versions(): healthy = False summary.append("some versions are unrecoverable") report.append("Unhealthy: some versions are unrecoverable") if len(recoverable) == 0: healthy = False summary.append("no versions are recoverable") report.append("Unhealthy: no versions are recoverable") if len(recoverable) > 1: healthy = False summary.append("multiple versions are recoverable") report.append("Unhealthy: there are multiple recoverable versions") if recoverable: best_version = smap.best_recoverable_version() report.append("Best Recoverable Version: " + smap.summarize_version(best_version)) counters = self._count_shares(smap, best_version) s = counters["count-shares-good"] k = counters["count-shares-needed"] N = counters["count-shares-expected"] if s < N: healthy = False report.append("Unhealthy: best version has only %d shares " "(encoding is %d-of-%d)" % (s, k, N)) summary.append("%d shares (enc %d-of-%d)" % (s, k, N)) elif unrecoverable: healthy = False # find a k and N from somewhere first = list(unrecoverable)[0] # not exactly the best version, but that doesn't matter too much counters = self._count_shares(smap, first) else: # couldn't find anything at all counters = { "count-shares-good": 0, "count-shares-needed": 3, # arbitrary defaults "count-shares-expected": 10, "count-good-share-hosts": 0, "count-wrong-shares": 0, } corrupt_share_locators = [] problems = [] if self.bad_shares: report.append("Corrupt Shares:") summary.append("Corrupt Shares:") for (server, shnum, f) in sorted(self.bad_shares): serverid = server.get_serverid() locator = (server, self._storage_index, shnum) corrupt_share_locators.append(locator) s = "%s-sh%d" % (server.get_name(), shnum) if f.check(CorruptShareError): ft = f.value.reason else: ft = str(f) report.append(" %s: %s" % (s, ft)) summary.append(s) p = (serverid, self._storage_index, shnum, f) problems.append(p) msg = ("CorruptShareError during mutable verify, " "serverid=%(serverid)s, si=%(si)s, shnum=%(shnum)d, " "where=%(where)s") log.msg(format=msg, serverid=server.get_name(), si=base32.b2a(self._storage_index), shnum=shnum, where=ft, level=log.WEIRD, umid="EkK8QA") sharemap = dictutil.DictOfSets() for verinfo in vmap: for (shnum, server, timestamp) in vmap[verinfo]: shareid = "%s-sh%d" % (smap.summarize_version(verinfo), shnum) sharemap.add(shareid, server) if healthy: summary = "Healthy" else: summary = "Unhealthy: " + " ".join(summary) count_happiness = servers_of_happiness(sharemap) cr = CheckResults(from_string(self._node.get_uri()), self._storage_index, healthy=healthy, recoverable=bool(recoverable), count_happiness=count_happiness, count_shares_needed=counters["count-shares-needed"], count_shares_expected=counters["count-shares-expected"], count_shares_good=counters["count-shares-good"], count_good_share_hosts=counters["count-good-share-hosts"], count_recoverable_versions=len(recoverable), count_unrecoverable_versions=len(unrecoverable), servers_responding=list(smap.get_reachable_servers()), sharemap=sharemap, count_wrong_shares=counters["count-wrong-shares"], list_corrupt_shares=corrupt_share_locators, count_corrupt_shares=len(corrupt_share_locators), list_incompatible_shares=[], count_incompatible_shares=0, summary=summary, report=report, share_problems=problems, servermap=smap.copy()) return cr
def _format_results(self, results): cr = CheckResults(self._verifycap, self._verifycap.get_storage_index()) d = {} d['count-shares-needed'] = self._verifycap.needed_shares d['count-shares-expected'] = self._verifycap.total_shares verifiedshares = dictutil.DictOfSets() # {sharenum: set(serverid)} servers = {} # {serverid: set(sharenums)} corruptsharelocators = [] # (serverid, storageindex, sharenum) incompatiblesharelocators = [] # (serverid, storageindex, sharenum) for theseverifiedshares, thisserver, thesecorruptshares, theseincompatibleshares, thisresponded in results: thisserverid = thisserver.get_serverid() servers.setdefault(thisserverid, set()).update(theseverifiedshares) for sharenum in theseverifiedshares: verifiedshares.setdefault(sharenum, set()).add(thisserverid) for sharenum in thesecorruptshares: corruptsharelocators.append( (thisserverid, self._verifycap.get_storage_index(), sharenum)) for sharenum in theseincompatibleshares: incompatiblesharelocators.append( (thisserverid, self._verifycap.get_storage_index(), sharenum)) d['count-shares-good'] = len(verifiedshares) d['count-good-share-hosts'] = len( [s for s in servers.keys() if servers[s]]) assert len(verifiedshares) <= self._verifycap.total_shares, ( verifiedshares.keys(), self._verifycap.total_shares) if len(verifiedshares) == self._verifycap.total_shares: cr.set_healthy(True) cr.set_summary("Healthy") else: cr.set_healthy(False) cr.set_summary("Not Healthy: %d shares (enc %d-of-%d)" % (len(verifiedshares), self._verifycap.needed_shares, self._verifycap.total_shares)) if len(verifiedshares) >= self._verifycap.needed_shares: cr.set_recoverable(True) d['count-recoverable-versions'] = 1 d['count-unrecoverable-versions'] = 0 else: cr.set_recoverable(False) d['count-recoverable-versions'] = 0 d['count-unrecoverable-versions'] = 1 d['servers-responding'] = list(servers) d['sharemap'] = verifiedshares # no such thing as wrong shares of an immutable file d['count-wrong-shares'] = 0 d['list-corrupt-shares'] = corruptsharelocators d['count-corrupt-shares'] = len(corruptsharelocators) d['list-incompatible-shares'] = incompatiblesharelocators d['count-incompatible-shares'] = len(incompatiblesharelocators) # The file needs rebalancing if the set of servers that have at least # one share is less than the number of uniquely-numbered shares # available. cr.set_needs_rebalancing( d['count-good-share-hosts'] < d['count-shares-good']) cr.set_data(d) return cr
def _format_results(self, results): SI = self._verifycap.get_storage_index() verifiedshares = dictutil.DictOfSets() # {sharenum: set(server)} servers = {} # {server: set(sharenums)} corruptshare_locators = [] # (server, storageindex, sharenum) incompatibleshare_locators = [] # (server, storageindex, sharenum) servers_responding = set() # server for verified, server, corrupt, incompatible, responded in results: servers.setdefault(server, set()).update(verified) for sharenum in verified: verifiedshares.setdefault(sharenum, set()).add(server) for sharenum in corrupt: corruptshare_locators.append((server, SI, sharenum)) for sharenum in incompatible: incompatibleshare_locators.append((server, SI, sharenum)) if responded: servers_responding.add(server) good_share_hosts = len([s for s in servers.keys() if servers[s]]) assert len(verifiedshares) <= self._verifycap.total_shares, ( verifiedshares.keys(), self._verifycap.total_shares) if len(verifiedshares) == self._verifycap.total_shares: healthy = True summary = "Healthy" else: healthy = False summary = ("Not Healthy: %d shares (enc %d-of-%d)" % (len(verifiedshares), self._verifycap.needed_shares, self._verifycap.total_shares)) if len(verifiedshares) >= self._verifycap.needed_shares: recoverable = 1 unrecoverable = 0 else: recoverable = 0 unrecoverable = 1 count_happiness = servers_of_happiness(verifiedshares) cr = CheckResults( self._verifycap, SI, healthy=healthy, recoverable=bool(recoverable), count_happiness=count_happiness, count_shares_needed=self._verifycap.needed_shares, count_shares_expected=self._verifycap.total_shares, count_shares_good=len(verifiedshares), count_good_share_hosts=good_share_hosts, count_recoverable_versions=recoverable, count_unrecoverable_versions=unrecoverable, servers_responding=list(servers_responding), sharemap=verifiedshares, count_wrong_shares=0, # no such thing, for immutable list_corrupt_shares=corruptshare_locators, count_corrupt_shares=len(corruptshare_locators), list_incompatible_shares=incompatibleshare_locators, count_incompatible_shares=len(incompatibleshare_locators), summary=summary, report=[], share_problems=[], servermap=None) return cr
def _render_results(self, ctx, cr): assert ICheckResults(cr) c = self.client sb = c.get_storage_broker() r = [] def add(name, value): r.append(T.li[name + ": ", value]) add("Report", T.pre["\n".join(self._html(cr.get_report()))]) add( "Share Counts", "need %d-of-%d, have %d" % (cr.get_encoding_needed(), cr.get_encoding_expected(), cr.get_share_counter_good())) add("Hosts with good shares", cr.get_host_counter_good_shares()) if cr.get_corrupt_shares(): badsharemap = [] for (s, si, shnum) in cr.get_corrupt_shares(): d = T.tr[T.td["sh#%d" % shnum], T.td[T.div(class_="nickname")[s.get_nickname()], T.div(class_="nodeid")[T.tt[s.get_name()]]], ] badsharemap.append(d) add( "Corrupt shares", T.table()[T.tr[T.th["Share ID"], T.th(class_="nickname-and-peerid")[ T.div["Nickname"], T.div(class_="nodeid")["Node ID"]]], badsharemap]) else: add("Corrupt shares", "none") add("Wrong Shares", cr.get_share_counter_wrong()) sharemap_data = [] shares_on_server = dictutil.DictOfSets() # FIXME: The two tables below contain nickname-and-nodeid table column markup which is duplicated with each other, introducer.xhtml, and deep-check-results.xhtml. All of these (and any other presentations of nickname-and-nodeid) should be combined. for shareid in sorted(cr.get_sharemap().keys()): servers = sorted(cr.get_sharemap()[shareid], key=lambda s: s.get_longname()) for i, s in enumerate(servers): shares_on_server.add(s, shareid) shareid_s = "" if i == 0: shareid_s = shareid d = T.tr[T.td[shareid_s], T.td[T.div(class_="nickname")[s.get_nickname()], T.div(class_="nodeid")[T.tt[s.get_name()]]]] sharemap_data.append(d) add( "Good Shares (sorted in share order)", T.table()[T.tr[T.th["Share ID"], T.th(class_="nickname-and-peerid")[ T.div["Nickname"], T.div(class_="nodeid")["Node ID"]]], sharemap_data]) add("Recoverable Versions", cr.get_version_counter_recoverable()) add("Unrecoverable Versions", cr.get_version_counter_unrecoverable()) # this table is sorted by permuted order permuted_servers = [ s for s in sb.get_servers_for_psi(cr.get_storage_index()) ] num_shares_left = sum( [len(shareids) for shareids in shares_on_server.values()]) servermap = [] for s in permuted_servers: shareids = list(shares_on_server.get(s, [])) shareids.reverse() shareids_s = [T.tt[shareid, " "] for shareid in sorted(shareids)] d = T.tr[T.td[T.div(class_="nickname")[s.get_nickname()], T.div(class_="nodeid")[T.tt[s.get_name()]]], T.td[shareids_s], ] servermap.append(d) num_shares_left -= len(shareids) if not num_shares_left: break add( "Share Balancing (servers in permuted order)", T.table()[T.tr[T.th( class_="nickname-and-peerid")[T.div["Nickname"], T.div( class_="nodeid")["Node ID"]], T.th["Share IDs"]], servermap]) return T.ul[r]
def _render_results(self, req, cr): assert ICheckResults(cr) c = self._client sb = c.get_storage_broker() r = [] def add(name, value): r.append(tags.li(name + ": ", value)) add("Report", tags.pre("\n".join(self._html(cr.get_report())))) add( "Share Counts", "need %d-of-%d, have %d" % (cr.get_encoding_needed(), cr.get_encoding_expected(), cr.get_share_counter_good())) add("Happiness Level", str(cr.get_happiness())) add("Hosts with good shares", str(cr.get_host_counter_good_shares())) if cr.get_corrupt_shares(): badsharemap = [] for (s, si, shnum) in cr.get_corrupt_shares(): d = tags.tr( tags.td("sh#%d" % shnum), tags.td(tags.div(s.get_nickname(), class_="nickname"), tags.div(tags.tt(s.get_name()), class_="nodeid")), ) badsharemap.append(d) add( "Corrupt shares", tags.table( tags.tr( tags.th("Share ID"), tags.th((tags.div("Nickname"), tags.div("Node ID", class_="nodeid")), class_="nickname-and-peerid")), badsharemap)) else: add("Corrupt shares", "none") add("Wrong Shares", str(cr.get_share_counter_wrong())) sharemap_data = [] shares_on_server = dictutil.DictOfSets() # FIXME: The two tables below contain nickname-and-nodeid # table column markup which is duplicated with each other, # introducer.xhtml, and deep-check-results.xhtml. All of these # (and any other presentations of nickname-and-nodeid) should be combined. for shareid in sorted(cr.get_sharemap().keys()): servers = sorted(cr.get_sharemap()[shareid], key=lambda s: s.get_longname()) for i, s in enumerate(servers): shares_on_server.add(s, shareid) shareid_s = "" if i == 0: if isinstance(shareid, bytes): shareid_s = str(shareid, "utf-8") else: shareid_s = str(shareid) d = tags.tr( tags.td(shareid_s), tags.td(tags.div(s.get_nickname(), class_="nickname"), tags.div(tags.tt(s.get_name()), class_="nodeid"))) sharemap_data.append(d) add( "Good Shares (sorted in share order)", tags.table( tags.tr( tags.th("Share ID"), tags.th(tags.div("Nickname"), tags.div("Node ID", class_="nodeid"), class_="nickname-and-peerid")), sharemap_data)) add("Recoverable Versions", str(cr.get_version_counter_recoverable())) add("Unrecoverable Versions", str(cr.get_version_counter_unrecoverable())) # this table is sorted by permuted order permuted_servers = [ s for s in sb.get_servers_for_psi(cr.get_storage_index()) ] num_shares_left = sum( [len(shareids) for shareids in shares_on_server.values()]) servermap = [] for s in permuted_servers: shareids = list(shares_on_server.get(s, [])) shareids.reverse() shareids_s = [ tags.tt(str(shareid), " ") for shareid in sorted(shareids) ] d = tags.tr( tags.td(tags.div(s.get_nickname(), class_="nickname"), tags.div(tags.tt(s.get_name()), class_="nodeid")), tags.td(shareids_s), ) servermap.append(d) num_shares_left -= len(shareids) if not num_shares_left: break add( "Share Balancing (servers in permuted order)", tags.table( tags.tr( tags.th(tags.div("Nickname"), tags.div("Node ID", class_="nodeid"), class_="nickname-and-peerid"), tags.th("Share IDs")), servermap)) return tags.ul(r)