def add_check(self, r, path):
if not r:
return # non-distributed object, i.e. LIT file
r = ICheckResults(r)
assert isinstance(path, (list, tuple))
self.objects_checked += 1
if r.is_healthy():
self.objects_healthy += 1
else:
self.objects_unhealthy += 1
if not r.is_recoverable():
self.objects_unrecoverable += 1
self.all_results[tuple(path)] = r
self.all_results_by_storage_index[r.get_storage_index()] = r
self.corrupt_shares.extend(r.get_data()["list-corrupt-shares"])
def __init__(self, node, check_results, storage_broker, history, monitor):
self.node = node
self.check_results = ICheckResults(check_results)
assert check_results.get_storage_index() == node.get_storage_index()
self._storage_broker = storage_broker
self._history = history
self._monitor = monitor
def data_problems(self, ctx, data):
all_objects = self.monitor.get_status().get_all_results()
for path in sorted(all_objects.keys()):
cr = all_objects[path]
assert ICheckResults.providedBy(cr)
if not cr.is_healthy():
yield path, cr
def data_problems(self, ctx, data):
all_objects = self.monitor.get_status().get_all_results()
for path in sorted(all_objects.keys()):
cr = all_objects[path]
assert ICheckResults.providedBy(cr)
if not cr.is_healthy():
yield path, cr
def __init__(self, client, results):
"""
:param allmydata.interfaces.IStatsProducer client: stats provider.
:param allmydata.interfaces.ICheckResults results: results of check/vefify operation.
"""
super(CheckResultsRenderer, self).__init__()
self._client = client
self._results = ICheckResults(results)
def add_check(self, r, path):
if not r:
return # non-distributed object, i.e. LIT file
r = ICheckResults(r)
assert isinstance(path, (list, tuple))
self.objects_checked += 1
if r.is_healthy():
self.objects_healthy += 1
else:
self.objects_unhealthy += 1
if not r.is_recoverable():
self.objects_unrecoverable += 1
self.all_results[tuple(path)] = r
self.all_results_by_storage_index[r.get_storage_index()] = r
self.corrupt_shares.extend(r.get_corrupt_shares())
def problems(self, req, tag):
all_objects = self.monitor.get_status().get_all_results()
problems = []
for path in sorted(all_objects.keys()):
cr = all_objects[path]
assert ICheckResults.providedBy(cr)
if not cr.is_healthy():
summary_text = ""
summary = cr.get_summary()
if summary:
summary_text = ": " + summary
summary_text += " [SI: %s]" % cr.get_storage_index_string()
problems.append({
# Not sure self._join_pathstring(path) is the
# right thing to use here.
"problem":
self._join_pathstring(path) + self._html(summary_text),
})
return SlotsSequenceElement(tag, problems)
class Repairer:
def __init__(self, node, check_results):
self.node = node
self.check_results = ICheckResults(check_results)
assert check_results.storage_index == self.node.get_storage_index()
def start(self, force=False):
# download, then re-publish. If a server had a bad share, try to
# replace it with a good one of the same shnum.
# The normal repair operation should not be used to replace
# application-specific merging of alternate versions: i.e if there
# are multiple highest seqnums with different roothashes. In this
# case, the application must use node.upload() (referencing the
# servermap that indicates the multiple-heads condition), or
# node.overwrite(). The repair() operation will refuse to run in
# these conditions unless a force=True argument is provided. If
# force=True is used, then the highest root hash will be reinforced.
# Likewise, the presence of an unrecoverable latest version is an
# unusual event, and should ideally be handled by retrying a couple
# times (spaced out over hours or days) and hoping that new shares
# will become available. If repair(force=True) is called, data will
# be lost: a new seqnum will be generated with the same contents as
# the most recent recoverable version, skipping over the lost
# version. repair(force=False) will refuse to run in a situation like
# this.
# Repair is designed to fix the following injuries:
# missing shares: add new ones to get at least N distinct ones
# old shares: replace old shares with the latest version
# bogus shares (bad sigs): replace the bad one with a good one
smap = self.check_results.get_servermap()
best_version = smap.best_recoverable_version()
if not best_version:
# the file is damaged beyond repair
rr = RepairResults(smap)
rr.set_successful(False)
return defer.succeed(rr)
if smap.unrecoverable_newer_versions():
if not force:
raise MustForceRepairError("There were unrecoverable newer "
"versions, so force=True must be "
"passed to the repair() operation")
# continuing on means that node.upload() will pick a seqnum that
# is higher than everything visible in the servermap, effectively
# discarding the unrecoverable versions.
if smap.needs_merge():
if not force:
raise MustForceRepairError("There were multiple recoverable "
"versions with identical seqnums, "
"so force=True must be passed to "
"the repair() operation")
# continuing on means that smap.best_recoverable_version() will
# pick the one with the highest roothash, and then node.upload()
# will replace all shares with its contents
# missing shares are handled during upload, which tries to find a
# home for every share
# old shares are handled during upload, which will replace any share
# that was present in the servermap
# bogus shares need to be managed here. We might notice a bogus share
# during mapupdate (whether done for a filecheck or just before a
# download) by virtue of it having an invalid signature. We might
# also notice a bad hash in the share during verify or download. In
# either case, the problem will be noted in the servermap, and the
# bad share (along with its checkstring) will be recorded in
# servermap.bad_shares . Publish knows that it should try and replace
# these.
# I chose to use the retrieve phase to ensure that the privkey is
# available, to avoid the extra roundtrip that would occur if we,
# say, added an smap.get_privkey() method.
if not self.node.get_writekey():
raise RepairRequiresWritecapError("Sorry, repair currently requires a writecap, to set the write-enabler properly.")
d = self.node.download_version(smap, best_version, fetch_privkey=True)
d.addCallback(self.node.upload, smap)
d.addCallback(self.get_results, smap)
return d
def get_results(self, res, smap):
rr = RepairResults(smap)
rr.set_successful(True)
return rr
def _render_results(self, req, cr):
assert ICheckResults(cr)
c = self._client
sb = c.get_storage_broker()
r = []
def add(name, value):
r.append(tags.li(name + ": ", value))
add("Report", tags.pre("\n".join(self._html(cr.get_report()))))
add(
"Share Counts", "need %d-of-%d, have %d" %
(cr.get_encoding_needed(), cr.get_encoding_expected(),
cr.get_share_counter_good()))
add("Happiness Level", str(cr.get_happiness()))
add("Hosts with good shares", str(cr.get_host_counter_good_shares()))
if cr.get_corrupt_shares():
badsharemap = []
for (s, si, shnum) in cr.get_corrupt_shares():
d = tags.tr(
tags.td("sh#%d" % shnum),
tags.td(tags.div(s.get_nickname(), class_="nickname"),
tags.div(tags.tt(s.get_name()), class_="nodeid")),
)
badsharemap.append(d)
add(
"Corrupt shares",
tags.table(
tags.tr(
tags.th("Share ID"),
tags.th((tags.div("Nickname"),
tags.div("Node ID", class_="nodeid")),
class_="nickname-and-peerid")), badsharemap))
else:
add("Corrupt shares", "none")
add("Wrong Shares", str(cr.get_share_counter_wrong()))
sharemap_data = []
shares_on_server = dictutil.DictOfSets()
# FIXME: The two tables below contain nickname-and-nodeid
# table column markup which is duplicated with each other,
# introducer.xhtml, and deep-check-results.xhtml. All of these
# (and any other presentations of nickname-and-nodeid) should be combined.
for shareid in sorted(cr.get_sharemap().keys()):
servers = sorted(cr.get_sharemap()[shareid],
key=lambda s: s.get_longname())
for i, s in enumerate(servers):
shares_on_server.add(s, shareid)
shareid_s = ""
if i == 0:
if isinstance(shareid, bytes):
shareid_s = str(shareid, "utf-8")
else:
shareid_s = str(shareid)
d = tags.tr(
tags.td(shareid_s),
tags.td(tags.div(s.get_nickname(), class_="nickname"),
tags.div(tags.tt(s.get_name()), class_="nodeid")))
sharemap_data.append(d)
add(
"Good Shares (sorted in share order)",
tags.table(
tags.tr(
tags.th("Share ID"),
tags.th(tags.div("Nickname"),
tags.div("Node ID", class_="nodeid"),
class_="nickname-and-peerid")), sharemap_data))
add("Recoverable Versions", str(cr.get_version_counter_recoverable()))
add("Unrecoverable Versions",
str(cr.get_version_counter_unrecoverable()))
# this table is sorted by permuted order
permuted_servers = [
s for s in sb.get_servers_for_psi(cr.get_storage_index())
]
num_shares_left = sum(
[len(shareids) for shareids in shares_on_server.values()])
servermap = []
for s in permuted_servers:
shareids = list(shares_on_server.get(s, []))
shareids.reverse()
shareids_s = [
tags.tt(str(shareid), " ") for shareid in sorted(shareids)
]
d = tags.tr(
tags.td(tags.div(s.get_nickname(), class_="nickname"),
tags.div(tags.tt(s.get_name()), class_="nodeid")),
tags.td(shareids_s),
)
servermap.append(d)
num_shares_left -= len(shareids)
if not num_shares_left:
break
add(
"Share Balancing (servers in permuted order)",
tags.table(
tags.tr(
tags.th(tags.div("Nickname"),
tags.div("Node ID", class_="nodeid"),
class_="nickname-and-peerid"),
tags.th("Share IDs")), servermap))
return tags.ul(r)
def repair(self, check_results, force=False, monitor=None):
assert ICheckResults(check_results)
r = Repairer(self, check_results, self._storage_broker,
self._history, monitor)
d = r.start(force)
return d
def __init__(self, node, check_results):
self.node = node
self.check_results = ICheckResults(check_results)
assert check_results.storage_index == self.node.get_storage_index()
def _render_results(self, ctx, cr):
assert ICheckResults(cr)
c = self.client
sb = c.get_storage_broker()
data = cr.get_data()
r = []
def add(name, value):
r.append(T.li[name + ": ", value])
add("Report", T.pre["\n".join(self._html(cr.get_report()))])
add(
"Share Counts", "need %d-of-%d, have %d" %
(data["count-shares-needed"], data["count-shares-expected"],
data["count-shares-good"]))
add("Hosts with good shares", data["count-good-share-hosts"])
if data["list-corrupt-shares"]:
badsharemap = []
for (serverid, si, shnum) in data["list-corrupt-shares"]:
nickname = sb.get_nickname_for_serverid(serverid)
badsharemap.append(T.tr[T.td["sh#%d" % shnum], T.td[
T.div(class_="nickname")[nickname],
T.div(class_="nodeid")[T.tt[base32.b2a(serverid)]]], ])
add(
"Corrupt shares",
T.table()[T.tr[T.th["Share ID"],
T.th(class_="nickname-and-peerid")[
T.div["Nickname"],
T.div(class_="nodeid")["Node ID"]]],
badsharemap])
else:
add("Corrupt shares", "none")
add("Wrong Shares", data["count-wrong-shares"])
sharemap = []
servers = {}
# FIXME: The two tables below contain nickname-and-nodeid table column markup which is duplicated with each other, introducer.xhtml, and deep-check-results.xhtml. All of these (and any other presentations of nickname-and-nodeid) should be combined.
for shareid in sorted(data["sharemap"].keys()):
serverids = data["sharemap"][shareid]
for i, serverid in enumerate(serverids):
if serverid not in servers:
servers[serverid] = []
servers[serverid].append(shareid)
shareid_s = ""
if i == 0:
shareid_s = shareid
nickname = sb.get_nickname_for_serverid(serverid)
sharemap.append(T.tr[T.td[shareid_s], T.td[
T.div(class_="nickname")[nickname],
T.div(class_="nodeid")[T.tt[base32.b2a(serverid)]]]])
add(
"Good Shares (sorted in share order)",
T.table()[T.tr[T.th["Share ID"],
T.th(class_="nickname-and-peerid")[
T.div["Nickname"],
T.div(class_="nodeid")["Node ID"]]], sharemap])
add("Recoverable Versions", data["count-recoverable-versions"])
add("Unrecoverable Versions", data["count-unrecoverable-versions"])
# this table is sorted by permuted order
sb = c.get_storage_broker()
permuted_servers = [
s for s in sb.get_servers_for_psi(cr.get_storage_index())
]
num_shares_left = sum([len(shares) for shares in servers.values()])
servermap = []
for s in permuted_servers:
nickname = s.get_nickname()
shareids = servers.get(s.get_serverid(), [])
shareids.reverse()
shareids_s = [T.tt[shareid, " "] for shareid in sorted(shareids)]
servermap.append(
T.tr[T.td[T.div(class_="nickname")[nickname],
T.div(class_="nodeid")[T.tt[s.get_name()]]],
T.td[shareids_s], ])
num_shares_left -= len(shareids)
if not num_shares_left:
break
add(
"Share Balancing (servers in permuted order)",
T.table()[T.tr[T.th(
class_="nickname-and-peerid")[T.div["Nickname"],
T.div(
class_="nodeid")["Node ID"]],
T.th["Share IDs"]], servermap])
return T.ul[r]
class Repairer:
def __init__(self, node, check_results):
self.node = node
self.check_results = ICheckResults(check_results)
assert check_results.storage_index == self.node.get_storage_index()
def start(self, force=False):
# download, then re-publish. If a server had a bad share, try to
# replace it with a good one of the same shnum.
# The normal repair operation should not be used to replace
# application-specific merging of alternate versions: i.e if there
# are multiple highest seqnums with different roothashes. In this
# case, the application must use node.upload() (referencing the
# servermap that indicates the multiple-heads condition), or
# node.overwrite(). The repair() operation will refuse to run in
# these conditions unless a force=True argument is provided. If
# force=True is used, then the highest root hash will be reinforced.
# Likewise, the presence of an unrecoverable latest version is an
# unusual event, and should ideally be handled by retrying a couple
# times (spaced out over hours or days) and hoping that new shares
# will become available. If repair(force=True) is called, data will
# be lost: a new seqnum will be generated with the same contents as
# the most recent recoverable version, skipping over the lost
# version. repair(force=False) will refuse to run in a situation like
# this.
# Repair is designed to fix the following injuries:
# missing shares: add new ones to get at least N distinct ones
# old shares: replace old shares with the latest version
# bogus shares (bad sigs): replace the bad one with a good one
smap = self.check_results.get_servermap()
best_version = smap.best_recoverable_version()
if not best_version:
# the file is damaged beyond repair
rr = RepairResults(smap)
rr.set_successful(False)
return defer.succeed(rr)
if smap.unrecoverable_newer_versions():
if not force:
raise MustForceRepairError("There were unrecoverable newer "
"versions, so force=True must be "
"passed to the repair() operation")
# continuing on means that node.upload() will pick a seqnum that
# is higher than everything visible in the servermap, effectively
# discarding the unrecoverable versions.
if smap.needs_merge():
if not force:
raise MustForceRepairError("There were multiple recoverable "
"versions with identical seqnums, "
"so force=True must be passed to "
"the repair() operation")
# continuing on means that smap.best_recoverable_version() will
# pick the one with the highest roothash, and then node.upload()
# will replace all shares with its contents
# missing shares are handled during upload, which tries to find a
# home for every share
# old shares are handled during upload, which will replace any share
# that was present in the servermap
# bogus shares need to be managed here. We might notice a bogus share
# during mapupdate (whether done for a filecheck or just before a
# download) by virtue of it having an invalid signature. We might
# also notice a bad hash in the share during verify or download. In
# either case, the problem will be noted in the servermap, and the
# bad share (along with its checkstring) will be recorded in
# servermap.bad_shares . Publish knows that it should try and replace
# these.
# I chose to use the retrieve phase to ensure that the privkey is
# available, to avoid the extra roundtrip that would occur if we,
# say, added an smap.get_privkey() method.
if not self.node.get_writekey():
raise RepairRequiresWritecapError(
"Sorry, repair currently requires a writecap, to set the write-enabler properly."
)
d = self.node.download_version(smap, best_version, fetch_privkey=True)
d.addCallback(lambda data: MutableData(data))
d.addCallback(self.node.upload, smap)
d.addCallback(self.get_results, smap)
return d
def get_results(self, res, smap):
rr = RepairResults(smap)
rr.set_successful(True)
return rr
def _render_results(self, ctx, cr):
assert ICheckResults(cr)
c = self.client
sb = c.get_storage_broker()
r = []
def add(name, value):
r.append(T.li[name + ": ", value])
add("Report", T.pre["\n".join(self._html(cr.get_report()))])
add(
"Share Counts", "need %d-of-%d, have %d" %
(cr.get_encoding_needed(), cr.get_encoding_expected(),
cr.get_share_counter_good()))
add("Hosts with good shares", cr.get_host_counter_good_shares())
if cr.get_corrupt_shares():
badsharemap = []
for (s, si, shnum) in cr.get_corrupt_shares():
d = T.tr[T.td["sh#%d" % shnum],
T.td[T.div(class_="nickname")[s.get_nickname()],
T.div(class_="nodeid")[T.tt[s.get_name()]]], ]
badsharemap.append(d)
add(
"Corrupt shares",
T.table()[T.tr[T.th["Share ID"],
T.th(class_="nickname-and-peerid")[
T.div["Nickname"],
T.div(class_="nodeid")["Node ID"]]],
badsharemap])
else:
add("Corrupt shares", "none")
add("Wrong Shares", cr.get_share_counter_wrong())
sharemap_data = []
shares_on_server = dictutil.DictOfSets()
# FIXME: The two tables below contain nickname-and-nodeid table column markup which is duplicated with each other, introducer.xhtml, and deep-check-results.xhtml. All of these (and any other presentations of nickname-and-nodeid) should be combined.
for shareid in sorted(cr.get_sharemap().keys()):
servers = sorted(cr.get_sharemap()[shareid],
key=lambda s: s.get_longname())
for i, s in enumerate(servers):
shares_on_server.add(s, shareid)
shareid_s = ""
if i == 0:
shareid_s = shareid
d = T.tr[T.td[shareid_s],
T.td[T.div(class_="nickname")[s.get_nickname()],
T.div(class_="nodeid")[T.tt[s.get_name()]]]]
sharemap_data.append(d)
add(
"Good Shares (sorted in share order)",
T.table()[T.tr[T.th["Share ID"],
T.th(class_="nickname-and-peerid")[
T.div["Nickname"],
T.div(class_="nodeid")["Node ID"]]],
sharemap_data])
add("Recoverable Versions", cr.get_version_counter_recoverable())
add("Unrecoverable Versions", cr.get_version_counter_unrecoverable())
# this table is sorted by permuted order
permuted_servers = [
s for s in sb.get_servers_for_psi(cr.get_storage_index())
]
num_shares_left = sum(
[len(shareids) for shareids in shares_on_server.values()])
servermap = []
for s in permuted_servers:
shareids = list(shares_on_server.get(s, []))
shareids.reverse()
shareids_s = [T.tt[shareid, " "] for shareid in sorted(shareids)]
d = T.tr[T.td[T.div(class_="nickname")[s.get_nickname()],
T.div(class_="nodeid")[T.tt[s.get_name()]]],
T.td[shareids_s], ]
servermap.append(d)
num_shares_left -= len(shareids)
if not num_shares_left:
break
add(
"Share Balancing (servers in permuted order)",
T.table()[T.tr[T.th(
class_="nickname-and-peerid")[T.div["Nickname"],
T.div(
class_="nodeid")["Node ID"]],
T.th["Share IDs"]], servermap])
return T.ul[r]
def __init__(self, node, check_results):
self.node = node
self.check_results = ICheckResults(check_results)
assert check_results.storage_index == self.node.get_storage_index()
def __init__(self, client, results):
self.client = client
self.r = ICheckResults(results)
rend.Page.__init__(self, results)
def repair(self, check_results, force=False):
assert ICheckResults(check_results)
r = Repairer(self, check_results)
d = r.start(force)
return d
