def marshal_json(s):
# common item data
item = {
"storage-index-string": base32.b2a_or_none(s.get_storage_index()),
"total-size": s.get_size(),
"status": s.get_status(),
}
# type-specific item date
if IUploadStatus.providedBy(s):
h, c, e = s.get_progress()
item["type"] = "upload"
item["progress-hash"] = h
item["progress-ciphertext"] = c
item["progress-encode-push"] = e
elif IDownloadStatus.providedBy(s):
item["type"] = "download"
item["progress"] = s.get_progress()
elif IPublishStatus.providedBy(s):
item["type"] = "publish"
elif IRetrieveStatus.providedBy(s):
item["type"] = "retrieve"
elif IServermapUpdaterStatus.providedBy(s):
item["type"] = "mapupdate"
item["mode"] = s.get_mode()
else:
item["type"] = "unknown"
item["class"] = s.__class__.__name__
return item
def json(self, req):
req.setHeader("content-type", "text/plain")
data = {}
data["active"] = active = []
for s in self._get_active_operations():
si_s = base32.b2a_or_none(s.get_storage_index())
size = s.get_size()
status = s.get_status()
if IUploadStatus.providedBy(s):
h,c,e = s.get_progress()
active.append({"type": "upload",
"storage-index-string": si_s,
"total-size": size,
"status": status,
"progress-hash": h,
"progress-ciphertext": c,
"progress-encode-push": e,
})
elif IDownloadStatus.providedBy(s):
active.append({"type": "download",
"storage-index-string": si_s,
"total-size": size,
"status": status,
"progress": s.get_progress(),
})
return simplejson.dumps(data, indent=1) + "\n"
def render_row(self, ctx, data):
s = data
TIME_FORMAT = "%H:%M:%S %d-%b-%Y"
started_s = time.strftime(TIME_FORMAT,
time.localtime(s.get_started()))
ctx.fillSlots("started", started_s)
si_s = base32.b2a_or_none(s.get_storage_index())
if si_s is None:
si_s = "(None)"
ctx.fillSlots("si", si_s)
ctx.fillSlots("helper", {True: "Yes",
False: "No"}[s.using_helper()])
size = s.get_size()
if size is None:
size = "(unknown)"
elif isinstance(size, (int, long, float)):
size = abbreviate_size(size)
ctx.fillSlots("total_size", size)
progress = data.get_progress()
if IUploadStatus.providedBy(data):
link = "up-%d" % data.get_counter()
ctx.fillSlots("type", "upload")
# TODO: make an ascii-art bar
(chk, ciphertext, encandpush) = progress
progress_s = ("hash: %.1f%%, ciphertext: %.1f%%, encode: %.1f%%" %
( (100.0 * chk),
(100.0 * ciphertext),
(100.0 * encandpush) ))
ctx.fillSlots("progress", progress_s)
elif IDownloadStatus.providedBy(data):
link = "down-%d" % data.get_counter()
ctx.fillSlots("type", "download")
ctx.fillSlots("progress", "%.1f%%" % (100.0 * progress))
elif IPublishStatus.providedBy(data):
link = "publish-%d" % data.get_counter()
ctx.fillSlots("type", "publish")
ctx.fillSlots("progress", "%.1f%%" % (100.0 * progress))
elif IRetrieveStatus.providedBy(data):
ctx.fillSlots("type", "retrieve")
link = "retrieve-%d" % data.get_counter()
ctx.fillSlots("progress", "%.1f%%" % (100.0 * progress))
else:
assert IServermapUpdaterStatus.providedBy(data)
ctx.fillSlots("type", "mapupdate %s" % data.get_mode())
link = "mapupdate-%d" % data.get_counter()
ctx.fillSlots("progress", "%.1f%%" % (100.0 * progress))
ctx.fillSlots("status", T.a(href=link)[s.get_status()])
return ctx.tag
def child_event_json(self, ctx):
inevow.IRequest(ctx).setHeader("content-type", "text/plain")
data = { } # this will be returned to the GET
ds = self.download_status
data.update(dict(
started=ds.get_started(),
storage_index=base32.b2a_or_none(ds.get_storage_index()),
helper=ds.using_helper(),
total_size=ds.get_size(),
progress=ds.get_progress(),
status=ds.get_status()))
data["read"] = self._find_overlap(ds.read_events,
"start_time", "finish_time")
data["segment"] = self._find_overlap(ds.segment_events,
"start_time", "finish_time")
data["dyhb"] = self._find_overlap(ds.dyhb_requests,
"start_time", "finish_time")
data["block"],data["block_rownums"] = self._find_overlap_requests(ds.block_requests)
servernums = {}
serverid_strings = {}
for d_ev in data["dyhb"]:
if d_ev["serverid"] not in servernums:
servernum = len(servernums)
servernums[d_ev["serverid"]] = servernum
#title= "%s: %s" % ( ",".join([str(shnum) for shnum in shnums]))
serverid_strings[servernum] = d_ev["serverid"][:4]
data["server_info"] = dict([(serverid, {"num": servernums[serverid],
"color": self.color(base32.a2b(serverid)),
"short": serverid_strings[servernums[serverid]],
})
for serverid in servernums.keys()])
data["num_serverids"] = len(serverid_strings)
# we'd prefer the keys of serverids[] to be ints, but this is JSON,
# so they get converted to strings. Stupid javascript.
data["serverids"] = serverid_strings
data["bounds"] = {"min": ds.first_timestamp, "max": ds.last_timestamp}
return simplejson.dumps(data, indent=1) + "\n"
def _got_data(self, results, blocknum):
precondition(blocknum < self.num_blocks,
self, blocknum, self.num_blocks)
sharehashes, blockhashes, blockdata = results
try:
sharehashes = dict(sharehashes)
except ValueError as le:
le.args = tuple(le.args + (sharehashes,))
raise
blockhashes = dict(enumerate(blockhashes))
candidate_share_hash = None # in case we log it in the except block below
blockhash = None # in case we log it in the except block below
try:
if self.share_hash_tree.needed_hashes(self.sharenum):
# This will raise exception if the values being passed do not
# match the root node of self.share_hash_tree.
try:
self.share_hash_tree.set_hashes(sharehashes)
except IndexError as le:
# Weird -- sharehashes contained index numbers outside of
# the range that fit into this hash tree.
raise BadOrMissingHash(le)
# To validate a block we need the root of the block hash tree,
# which is also one of the leafs of the share hash tree, and is
# called "the share hash".
if not self.block_hash_tree[0]: # empty -- no root node yet
# Get the share hash from the share hash tree.
share_hash = self.share_hash_tree.get_leaf(self.sharenum)
if not share_hash:
# No root node in block_hash_tree and also the share hash
# wasn't sent by the server.
raise hashtree.NotEnoughHashesError
self.block_hash_tree.set_hashes({0: share_hash})
if self.block_hash_tree.needed_hashes(blocknum):
self.block_hash_tree.set_hashes(blockhashes)
blockhash = block_hash(blockdata)
self.block_hash_tree.set_hashes(leaves={blocknum: blockhash})
#self.log("checking block_hash(shareid=%d, blocknum=%d) len=%d "
# "%r .. %r: %s" %
# (self.sharenum, blocknum, len(blockdata),
# blockdata[:50], blockdata[-50:], base32.b2a(blockhash)))
except (hashtree.BadHashError, hashtree.NotEnoughHashesError) as le:
# log.WEIRD: indicates undetected disk/network error, or more
# likely a programming error
self.log("hash failure in block=%d, shnum=%d on %s" %
(blocknum, self.sharenum, self.bucket))
if self.block_hash_tree.needed_hashes(blocknum):
self.log(""" failure occurred when checking the block_hash_tree.
This suggests that either the block data was bad, or that the
block hashes we received along with it were bad.""")
else:
self.log(""" the failure probably occurred when checking the
share_hash_tree, which suggests that the share hashes we
received from the remote peer were bad.""")
self.log(" have candidate_share_hash: %s" % bool(candidate_share_hash))
self.log(" block length: %d" % len(blockdata))
self.log(" block hash: %s" % base32.b2a_or_none(blockhash))
if len(blockdata) < 100:
self.log(" block data: %r" % (blockdata,))
else:
self.log(" block data start/end: %r .. %r" %
(blockdata[:50], blockdata[-50:]))
self.log(" share hash tree:\n" + self.share_hash_tree.dump())
self.log(" block hash tree:\n" + self.block_hash_tree.dump())
lines = []
for i,h in sorted(sharehashes.items()):
lines.append("%3d: %s" % (i, base32.b2a_or_none(h)))
self.log(" sharehashes:\n" + "\n".join(lines) + "\n")
lines = []
for i,h in blockhashes.items():
lines.append("%3d: %s" % (i, base32.b2a_or_none(h)))
log.msg(" blockhashes:\n" + "\n".join(lines) + "\n")
raise BadOrMissingHash(le)
# If we made it here, the block is good. If the hash trees didn't
# like what they saw, they would have raised a BadHashError, causing
# our caller to see a Failure and thus ignore this block (as well as
# dropping this bucket).
return blockdata
def render_si(self, ctx, data):
si_s = base32.b2a_or_none(data.get_storage_index())
if si_s is None:
si_s = "(None)"
return si_s
except (hashtree.BadHashError, hashtree.NotEnoughHashesError), le:
# log.WEIRD: indicates undetected disk/network error, or more
# likely a programming error
self.log("hash failure in block=%d, shnum=%d on %s" %
(blocknum, self.sharenum, self.bucket))
if self.block_hash_tree.needed_hashes(blocknum):
self.log(""" failure occurred when checking the block_hash_tree.
This suggests that either the block data was bad, or that the
block hashes we received along with it were bad.""")
else:
self.log(""" the failure probably occurred when checking the
share_hash_tree, which suggests that the share hashes we
received from the remote peer were bad.""")
self.log(" have candidate_share_hash: %s" % bool(candidate_share_hash))
self.log(" block length: %d" % len(blockdata))
self.log(" block hash: %s" % base32.b2a_or_none(blockhash))
if len(blockdata) < 100:
self.log(" block data: %r" % (blockdata,))
else:
self.log(" block data start/end: %r .. %r" %
(blockdata[:50], blockdata[-50:]))
self.log(" share hash tree:\n" + self.share_hash_tree.dump())
self.log(" block hash tree:\n" + self.block_hash_tree.dump())
lines = []
for i,h in sorted(sharehashes.items()):
lines.append("%3d: %s" % (i, base32.b2a_or_none(h)))
self.log(" sharehashes:\n" + "\n".join(lines) + "\n")
lines = []
for i,h in blockhashes.items():
lines.append("%3d: %s" % (i, base32.b2a_or_none(h)))
log.msg(" blockhashes:\n" + "\n".join(lines) + "\n")
def render_si(self, ctx, data):
si_s = base32.b2a_or_none(data.get_storage_index())
if si_s is None:
si_s = "(None)"
return si_s
# likely a programming error
self.log("hash failure in block=%d, shnum=%d on %s" %
(blocknum, self.sharenum, self.bucket))
if self.block_hash_tree.needed_hashes(blocknum):
self.log(
""" failure occurred when checking the block_hash_tree.
This suggests that either the block data was bad, or that the
block hashes we received along with it were bad.""")
else:
self.log(""" the failure probably occurred when checking the
share_hash_tree, which suggests that the share hashes we
received from the remote peer were bad.""")
self.log(" have candidate_share_hash: %s" %
bool(candidate_share_hash))
self.log(" block length: %d" % len(blockdata))
self.log(" block hash: %s" % base32.b2a_or_none(blockhash))
if len(blockdata) < 100:
self.log(" block data: %r" % (blockdata, ))
else:
self.log(" block data start/end: %r .. %r" %
(blockdata[:50], blockdata[-50:]))
self.log(" share hash tree:\n" + self.share_hash_tree.dump())
self.log(" block hash tree:\n" + self.block_hash_tree.dump())
lines = []
for i, h in sorted(sharehashes.items()):
lines.append("%3d: %s" % (i, base32.b2a_or_none(h)))
self.log(" sharehashes:\n" + "\n".join(lines) + "\n")
lines = []
for i, h in blockhashes.items():
lines.append("%3d: %s" % (i, base32.b2a_or_none(h)))
log.msg(" blockhashes:\n" + "\n".join(lines) + "\n")
def get_storage_index_string(self):
return base32.b2a_or_none(self._storage_index)
def _got_data(self, results, blocknum):
precondition(blocknum < self.num_blocks, self, blocknum,
self.num_blocks)
sharehashes, blockhashes, blockdata = results
try:
sharehashes = dict(sharehashes)
except ValueError as le:
le.args = tuple(le.args + (sharehashes, ))
raise
blockhashes = dict(enumerate(blockhashes))
candidate_share_hash = None # in case we log it in the except block below
blockhash = None # in case we log it in the except block below
try:
if self.share_hash_tree.needed_hashes(self.sharenum):
# This will raise exception if the values being passed do not
# match the root node of self.share_hash_tree.
try:
self.share_hash_tree.set_hashes(sharehashes)
except IndexError as le:
# Weird -- sharehashes contained index numbers outside of
# the range that fit into this hash tree.
raise BadOrMissingHash(le)
# To validate a block we need the root of the block hash tree,
# which is also one of the leafs of the share hash tree, and is
# called "the share hash".
if not self.block_hash_tree[0]: # empty -- no root node yet
# Get the share hash from the share hash tree.
share_hash = self.share_hash_tree.get_leaf(self.sharenum)
if not share_hash:
# No root node in block_hash_tree and also the share hash
# wasn't sent by the server.
raise hashtree.NotEnoughHashesError
self.block_hash_tree.set_hashes({0: share_hash})
if self.block_hash_tree.needed_hashes(blocknum):
self.block_hash_tree.set_hashes(blockhashes)
blockhash = block_hash(blockdata)
self.block_hash_tree.set_hashes(leaves={blocknum: blockhash})
#self.log("checking block_hash(shareid=%d, blocknum=%d) len=%d "
# "%r .. %r: %s" %
# (self.sharenum, blocknum, len(blockdata),
# blockdata[:50], blockdata[-50:], base32.b2a(blockhash)))
except (hashtree.BadHashError, hashtree.NotEnoughHashesError) as le:
# log.WEIRD: indicates undetected disk/network error, or more
# likely a programming error
self.log("hash failure in block=%d, shnum=%d on %s" %
(blocknum, self.sharenum, self.bucket))
if self.block_hash_tree.needed_hashes(blocknum):
self.log(
""" failure occurred when checking the block_hash_tree.
This suggests that either the block data was bad, or that the
block hashes we received along with it were bad.""")
else:
self.log(""" the failure probably occurred when checking the
share_hash_tree, which suggests that the share hashes we
received from the remote peer were bad.""")
self.log(" have candidate_share_hash: %s" %
bool(candidate_share_hash))
self.log(" block length: %d" % len(blockdata))
self.log(" block hash: %s" % base32.b2a_or_none(blockhash))
if len(blockdata) < 100:
self.log(" block data: %r" % (blockdata, ))
else:
self.log(" block data start/end: %r .. %r" %
(blockdata[:50], blockdata[-50:]))
self.log(" share hash tree:\n" + self.share_hash_tree.dump())
self.log(" block hash tree:\n" + self.block_hash_tree.dump())
lines = []
for i, h in sorted(sharehashes.items()):
lines.append("%3d: %s" % (i, base32.b2a_or_none(h)))
self.log(" sharehashes:\n" + "\n".join(lines) + "\n")
lines = []
for i, h in blockhashes.items():
lines.append("%3d: %s" % (i, base32.b2a_or_none(h)))
log.msg(" blockhashes:\n" + "\n".join(lines) + "\n")
raise BadOrMissingHash(le)
# If we made it here, the block is good. If the hash trees didn't
# like what they saw, they would have raised a BadHashError, causing
# our caller to see a Failure and thus ignore this block (as well as
# dropping this bucket).
return blockdata
def test_b2a_or_none(self):
self.failUnlessEqual(base32.b2a_or_none(None), None)
self.failUnlessEqual(base32.b2a_or_none(b"\x12\x34"), b"ci2a")
def dump(self):
lines = []
for i,depth in self.depth_first():
lines.append("%s%3d: %s" % (" "*depth, i,
base32.b2a_or_none(self[i])))
return "\n".join(lines) + "\n"
def si(self, req, tag):
si_s = base32.b2a_or_none(self._publish_status.get_storage_index())
if si_s is None:
si_s = "(None)"
return tag(str(si_s))
def dump(self):
lines = []
for i,depth in self.depth_first():
lines.append("%s%3d: %s" % (" "*depth, i,
base32.b2a_or_none(self[i])))
return "\n".join(lines) + "\n"
