def marshal_json(s): # common item data item = { "storage-index-string": base32.b2a_or_none(s.get_storage_index()), "total-size": s.get_size(), "status": s.get_status(), } # type-specific item date if IUploadStatus.providedBy(s): h, c, e = s.get_progress() item["type"] = "upload" item["progress-hash"] = h item["progress-ciphertext"] = c item["progress-encode-push"] = e elif IDownloadStatus.providedBy(s): item["type"] = "download" item["progress"] = s.get_progress() elif IPublishStatus.providedBy(s): item["type"] = "publish" elif IRetrieveStatus.providedBy(s): item["type"] = "retrieve" elif IServermapUpdaterStatus.providedBy(s): item["type"] = "mapupdate" item["mode"] = s.get_mode() else: item["type"] = "unknown" item["class"] = s.__class__.__name__ return item
def json(self, req): req.setHeader("content-type", "text/plain") data = {} data["active"] = active = [] for s in self._get_active_operations(): si_s = base32.b2a_or_none(s.get_storage_index()) size = s.get_size() status = s.get_status() if IUploadStatus.providedBy(s): h,c,e = s.get_progress() active.append({"type": "upload", "storage-index-string": si_s, "total-size": size, "status": status, "progress-hash": h, "progress-ciphertext": c, "progress-encode-push": e, }) elif IDownloadStatus.providedBy(s): active.append({"type": "download", "storage-index-string": si_s, "total-size": size, "status": status, "progress": s.get_progress(), }) return simplejson.dumps(data, indent=1) + "\n"
def render_row(self, ctx, data): s = data TIME_FORMAT = "%H:%M:%S %d-%b-%Y" started_s = time.strftime(TIME_FORMAT, time.localtime(s.get_started())) ctx.fillSlots("started", started_s) si_s = base32.b2a_or_none(s.get_storage_index()) if si_s is None: si_s = "(None)" ctx.fillSlots("si", si_s) ctx.fillSlots("helper", {True: "Yes", False: "No"}[s.using_helper()]) size = s.get_size() if size is None: size = "(unknown)" elif isinstance(size, (int, long, float)): size = abbreviate_size(size) ctx.fillSlots("total_size", size) progress = data.get_progress() if IUploadStatus.providedBy(data): link = "up-%d" % data.get_counter() ctx.fillSlots("type", "upload") # TODO: make an ascii-art bar (chk, ciphertext, encandpush) = progress progress_s = ("hash: %.1f%%, ciphertext: %.1f%%, encode: %.1f%%" % ( (100.0 * chk), (100.0 * ciphertext), (100.0 * encandpush) )) ctx.fillSlots("progress", progress_s) elif IDownloadStatus.providedBy(data): link = "down-%d" % data.get_counter() ctx.fillSlots("type", "download") ctx.fillSlots("progress", "%.1f%%" % (100.0 * progress)) elif IPublishStatus.providedBy(data): link = "publish-%d" % data.get_counter() ctx.fillSlots("type", "publish") ctx.fillSlots("progress", "%.1f%%" % (100.0 * progress)) elif IRetrieveStatus.providedBy(data): ctx.fillSlots("type", "retrieve") link = "retrieve-%d" % data.get_counter() ctx.fillSlots("progress", "%.1f%%" % (100.0 * progress)) else: assert IServermapUpdaterStatus.providedBy(data) ctx.fillSlots("type", "mapupdate %s" % data.get_mode()) link = "mapupdate-%d" % data.get_counter() ctx.fillSlots("progress", "%.1f%%" % (100.0 * progress)) ctx.fillSlots("status", T.a(href=link)[s.get_status()]) return ctx.tag
def child_event_json(self, ctx): inevow.IRequest(ctx).setHeader("content-type", "text/plain") data = { } # this will be returned to the GET ds = self.download_status data.update(dict( started=ds.get_started(), storage_index=base32.b2a_or_none(ds.get_storage_index()), helper=ds.using_helper(), total_size=ds.get_size(), progress=ds.get_progress(), status=ds.get_status())) data["read"] = self._find_overlap(ds.read_events, "start_time", "finish_time") data["segment"] = self._find_overlap(ds.segment_events, "start_time", "finish_time") data["dyhb"] = self._find_overlap(ds.dyhb_requests, "start_time", "finish_time") data["block"],data["block_rownums"] = self._find_overlap_requests(ds.block_requests) servernums = {} serverid_strings = {} for d_ev in data["dyhb"]: if d_ev["serverid"] not in servernums: servernum = len(servernums) servernums[d_ev["serverid"]] = servernum #title= "%s: %s" % ( ",".join([str(shnum) for shnum in shnums])) serverid_strings[servernum] = d_ev["serverid"][:4] data["server_info"] = dict([(serverid, {"num": servernums[serverid], "color": self.color(base32.a2b(serverid)), "short": serverid_strings[servernums[serverid]], }) for serverid in servernums.keys()]) data["num_serverids"] = len(serverid_strings) # we'd prefer the keys of serverids[] to be ints, but this is JSON, # so they get converted to strings. Stupid javascript. data["serverids"] = serverid_strings data["bounds"] = {"min": ds.first_timestamp, "max": ds.last_timestamp} return simplejson.dumps(data, indent=1) + "\n"
def _got_data(self, results, blocknum): precondition(blocknum < self.num_blocks, self, blocknum, self.num_blocks) sharehashes, blockhashes, blockdata = results try: sharehashes = dict(sharehashes) except ValueError as le: le.args = tuple(le.args + (sharehashes,)) raise blockhashes = dict(enumerate(blockhashes)) candidate_share_hash = None # in case we log it in the except block below blockhash = None # in case we log it in the except block below try: if self.share_hash_tree.needed_hashes(self.sharenum): # This will raise exception if the values being passed do not # match the root node of self.share_hash_tree. try: self.share_hash_tree.set_hashes(sharehashes) except IndexError as le: # Weird -- sharehashes contained index numbers outside of # the range that fit into this hash tree. raise BadOrMissingHash(le) # To validate a block we need the root of the block hash tree, # which is also one of the leafs of the share hash tree, and is # called "the share hash". if not self.block_hash_tree[0]: # empty -- no root node yet # Get the share hash from the share hash tree. share_hash = self.share_hash_tree.get_leaf(self.sharenum) if not share_hash: # No root node in block_hash_tree and also the share hash # wasn't sent by the server. raise hashtree.NotEnoughHashesError self.block_hash_tree.set_hashes({0: share_hash}) if self.block_hash_tree.needed_hashes(blocknum): self.block_hash_tree.set_hashes(blockhashes) blockhash = block_hash(blockdata) self.block_hash_tree.set_hashes(leaves={blocknum: blockhash}) #self.log("checking block_hash(shareid=%d, blocknum=%d) len=%d " # "%r .. %r: %s" % # (self.sharenum, blocknum, len(blockdata), # blockdata[:50], blockdata[-50:], base32.b2a(blockhash))) except (hashtree.BadHashError, hashtree.NotEnoughHashesError) as le: # log.WEIRD: indicates undetected disk/network error, or more # likely a programming error self.log("hash failure in block=%d, shnum=%d on %s" % (blocknum, self.sharenum, self.bucket)) if self.block_hash_tree.needed_hashes(blocknum): self.log(""" failure occurred when checking the block_hash_tree. This suggests that either the block data was bad, or that the block hashes we received along with it were bad.""") else: self.log(""" the failure probably occurred when checking the share_hash_tree, which suggests that the share hashes we received from the remote peer were bad.""") self.log(" have candidate_share_hash: %s" % bool(candidate_share_hash)) self.log(" block length: %d" % len(blockdata)) self.log(" block hash: %s" % base32.b2a_or_none(blockhash)) if len(blockdata) < 100: self.log(" block data: %r" % (blockdata,)) else: self.log(" block data start/end: %r .. %r" % (blockdata[:50], blockdata[-50:])) self.log(" share hash tree:\n" + self.share_hash_tree.dump()) self.log(" block hash tree:\n" + self.block_hash_tree.dump()) lines = [] for i,h in sorted(sharehashes.items()): lines.append("%3d: %s" % (i, base32.b2a_or_none(h))) self.log(" sharehashes:\n" + "\n".join(lines) + "\n") lines = [] for i,h in blockhashes.items(): lines.append("%3d: %s" % (i, base32.b2a_or_none(h))) log.msg(" blockhashes:\n" + "\n".join(lines) + "\n") raise BadOrMissingHash(le) # If we made it here, the block is good. If the hash trees didn't # like what they saw, they would have raised a BadHashError, causing # our caller to see a Failure and thus ignore this block (as well as # dropping this bucket). return blockdata
def render_si(self, ctx, data): si_s = base32.b2a_or_none(data.get_storage_index()) if si_s is None: si_s = "(None)" return si_s
except (hashtree.BadHashError, hashtree.NotEnoughHashesError), le: # log.WEIRD: indicates undetected disk/network error, or more # likely a programming error self.log("hash failure in block=%d, shnum=%d on %s" % (blocknum, self.sharenum, self.bucket)) if self.block_hash_tree.needed_hashes(blocknum): self.log(""" failure occurred when checking the block_hash_tree. This suggests that either the block data was bad, or that the block hashes we received along with it were bad.""") else: self.log(""" the failure probably occurred when checking the share_hash_tree, which suggests that the share hashes we received from the remote peer were bad.""") self.log(" have candidate_share_hash: %s" % bool(candidate_share_hash)) self.log(" block length: %d" % len(blockdata)) self.log(" block hash: %s" % base32.b2a_or_none(blockhash)) if len(blockdata) < 100: self.log(" block data: %r" % (blockdata,)) else: self.log(" block data start/end: %r .. %r" % (blockdata[:50], blockdata[-50:])) self.log(" share hash tree:\n" + self.share_hash_tree.dump()) self.log(" block hash tree:\n" + self.block_hash_tree.dump()) lines = [] for i,h in sorted(sharehashes.items()): lines.append("%3d: %s" % (i, base32.b2a_or_none(h))) self.log(" sharehashes:\n" + "\n".join(lines) + "\n") lines = [] for i,h in blockhashes.items(): lines.append("%3d: %s" % (i, base32.b2a_or_none(h))) log.msg(" blockhashes:\n" + "\n".join(lines) + "\n")
# likely a programming error self.log("hash failure in block=%d, shnum=%d on %s" % (blocknum, self.sharenum, self.bucket)) if self.block_hash_tree.needed_hashes(blocknum): self.log( """ failure occurred when checking the block_hash_tree. This suggests that either the block data was bad, or that the block hashes we received along with it were bad.""") else: self.log(""" the failure probably occurred when checking the share_hash_tree, which suggests that the share hashes we received from the remote peer were bad.""") self.log(" have candidate_share_hash: %s" % bool(candidate_share_hash)) self.log(" block length: %d" % len(blockdata)) self.log(" block hash: %s" % base32.b2a_or_none(blockhash)) if len(blockdata) < 100: self.log(" block data: %r" % (blockdata, )) else: self.log(" block data start/end: %r .. %r" % (blockdata[:50], blockdata[-50:])) self.log(" share hash tree:\n" + self.share_hash_tree.dump()) self.log(" block hash tree:\n" + self.block_hash_tree.dump()) lines = [] for i, h in sorted(sharehashes.items()): lines.append("%3d: %s" % (i, base32.b2a_or_none(h))) self.log(" sharehashes:\n" + "\n".join(lines) + "\n") lines = [] for i, h in blockhashes.items(): lines.append("%3d: %s" % (i, base32.b2a_or_none(h))) log.msg(" blockhashes:\n" + "\n".join(lines) + "\n")
def get_storage_index_string(self): return base32.b2a_or_none(self._storage_index)
def _got_data(self, results, blocknum): precondition(blocknum < self.num_blocks, self, blocknum, self.num_blocks) sharehashes, blockhashes, blockdata = results try: sharehashes = dict(sharehashes) except ValueError as le: le.args = tuple(le.args + (sharehashes, )) raise blockhashes = dict(enumerate(blockhashes)) candidate_share_hash = None # in case we log it in the except block below blockhash = None # in case we log it in the except block below try: if self.share_hash_tree.needed_hashes(self.sharenum): # This will raise exception if the values being passed do not # match the root node of self.share_hash_tree. try: self.share_hash_tree.set_hashes(sharehashes) except IndexError as le: # Weird -- sharehashes contained index numbers outside of # the range that fit into this hash tree. raise BadOrMissingHash(le) # To validate a block we need the root of the block hash tree, # which is also one of the leafs of the share hash tree, and is # called "the share hash". if not self.block_hash_tree[0]: # empty -- no root node yet # Get the share hash from the share hash tree. share_hash = self.share_hash_tree.get_leaf(self.sharenum) if not share_hash: # No root node in block_hash_tree and also the share hash # wasn't sent by the server. raise hashtree.NotEnoughHashesError self.block_hash_tree.set_hashes({0: share_hash}) if self.block_hash_tree.needed_hashes(blocknum): self.block_hash_tree.set_hashes(blockhashes) blockhash = block_hash(blockdata) self.block_hash_tree.set_hashes(leaves={blocknum: blockhash}) #self.log("checking block_hash(shareid=%d, blocknum=%d) len=%d " # "%r .. %r: %s" % # (self.sharenum, blocknum, len(blockdata), # blockdata[:50], blockdata[-50:], base32.b2a(blockhash))) except (hashtree.BadHashError, hashtree.NotEnoughHashesError) as le: # log.WEIRD: indicates undetected disk/network error, or more # likely a programming error self.log("hash failure in block=%d, shnum=%d on %s" % (blocknum, self.sharenum, self.bucket)) if self.block_hash_tree.needed_hashes(blocknum): self.log( """ failure occurred when checking the block_hash_tree. This suggests that either the block data was bad, or that the block hashes we received along with it were bad.""") else: self.log(""" the failure probably occurred when checking the share_hash_tree, which suggests that the share hashes we received from the remote peer were bad.""") self.log(" have candidate_share_hash: %s" % bool(candidate_share_hash)) self.log(" block length: %d" % len(blockdata)) self.log(" block hash: %s" % base32.b2a_or_none(blockhash)) if len(blockdata) < 100: self.log(" block data: %r" % (blockdata, )) else: self.log(" block data start/end: %r .. %r" % (blockdata[:50], blockdata[-50:])) self.log(" share hash tree:\n" + self.share_hash_tree.dump()) self.log(" block hash tree:\n" + self.block_hash_tree.dump()) lines = [] for i, h in sorted(sharehashes.items()): lines.append("%3d: %s" % (i, base32.b2a_or_none(h))) self.log(" sharehashes:\n" + "\n".join(lines) + "\n") lines = [] for i, h in blockhashes.items(): lines.append("%3d: %s" % (i, base32.b2a_or_none(h))) log.msg(" blockhashes:\n" + "\n".join(lines) + "\n") raise BadOrMissingHash(le) # If we made it here, the block is good. If the hash trees didn't # like what they saw, they would have raised a BadHashError, causing # our caller to see a Failure and thus ignore this block (as well as # dropping this bucket). return blockdata
def test_b2a_or_none(self): self.failUnlessEqual(base32.b2a_or_none(None), None) self.failUnlessEqual(base32.b2a_or_none(b"\x12\x34"), b"ci2a")
def dump(self): lines = [] for i,depth in self.depth_first(): lines.append("%s%3d: %s" % (" "*depth, i, base32.b2a_or_none(self[i]))) return "\n".join(lines) + "\n"
def si(self, req, tag): si_s = base32.b2a_or_none(self._publish_status.get_storage_index()) if si_s is None: si_s = "(None)" return tag(str(si_s))