def _describe_immutable_share(abs_sharefile, now, si_s, out): class ImmediateReadBucketProxy(ReadBucketProxy): def __init__(self, sf): self.sf = sf ReadBucketProxy.__init__(self, None, None, "") def __repr__(self): return "<ImmediateReadBucketProxy>" def _read(self, offset, size): return defer.succeed(sf.read_share_data(offset, size)) # use a ReadBucketProxy to parse the bucket and find the uri extension sf = ShareFile(abs_sharefile) bp = ImmediateReadBucketProxy(sf) expiration_time = min(lease.get_expiration_time() for lease in sf.get_leases()) expiration = max(0, expiration_time - now) UEB_data = call(bp.get_uri_extension) unpacked = uri.unpack_extension_readable(UEB_data) k = unpacked["needed_shares"] N = unpacked["total_shares"] filesize = unpacked["size"] ueb_hash = unpacked["UEB_hash"] print("CHK %s %d/%d %d %s %d %s" % (si_s, k, N, filesize, str( ueb_hash, "utf-8"), expiration, quote_output(abs_sharefile)), file=out)
def get_leases(self, storage_index): """Provide an iterator that yields all of the leases attached to this bucket. Each lease is returned as a LeaseInfo instance. This method is not for client use. """ # since all shares get the same lease data, we just grab the leases # from the first share try: shnum, filename = self._get_bucket_shares(storage_index).next() sf = ShareFile(filename) return sf.get_leases() except StopIteration: return iter([])
def describe_share(abs_sharefile, si_s, shnum_s, now, out): from allmydata import uri from allmydata.storage.mutable import MutableShareFile from allmydata.storage.immutable import ShareFile from allmydata.mutable.layout import unpack_share from allmydata.mutable.common import NeedMoreDataError from allmydata.immutable.layout import ReadBucketProxy from allmydata.util import base32 from allmydata.util.encodingutil import quote_output import struct f = open(abs_sharefile, "rb") prefix = f.read(32) if prefix == MutableShareFile.MAGIC: # mutable share m = MutableShareFile(abs_sharefile) WE, nodeid = m._read_write_enabler_and_nodeid(f) data_length = m._read_data_length(f) expiration_time = min( [lease.expiration_time for (i, lease) in m._enumerate_leases(f)]) expiration = max(0, expiration_time - now) share_type = "unknown" f.seek(m.DATA_OFFSET) version = f.read(1) if version == b"\x00": # this slot contains an SMDF share share_type = "SDMF" elif version == b"\x01": share_type = "MDMF" if share_type == "SDMF": f.seek(m.DATA_OFFSET) data = f.read(min(data_length, 2000)) try: pieces = unpack_share(data) except NeedMoreDataError as e: # retry once with the larger size size = e.needed_bytes f.seek(m.DATA_OFFSET) data = f.read(min(data_length, size)) pieces = unpack_share(data) (seqnum, root_hash, IV, k, N, segsize, datalen, pubkey, signature, share_hash_chain, block_hash_tree, share_data, enc_privkey) = pieces print("SDMF %s %d/%d %d #%d:%s %d %s" % \ (si_s, k, N, datalen, seqnum, unicode(base32.b2a(root_hash), "utf-8"), expiration, quote_output(abs_sharefile)), file=out) elif share_type == "MDMF": from allmydata.mutable.layout import MDMFSlotReadProxy fake_shnum = 0 # TODO: factor this out with dump_MDMF_share() class ShareDumper(MDMFSlotReadProxy): def _read(self, readvs, force_remote=False, queue=False): data = [] for (where, length) in readvs: f.seek(m.DATA_OFFSET + where) data.append(f.read(length)) return defer.succeed({fake_shnum: data}) p = ShareDumper(None, "fake-si", fake_shnum) def extract(func): stash = [] # these methods return Deferreds, but we happen to know that # they run synchronously when not actually talking to a # remote server d = func() d.addCallback(stash.append) return stash[0] verinfo = extract(p.get_verinfo) (seqnum, root_hash, salt_to_use, segsize, datalen, k, N, prefix, offsets) = verinfo print("MDMF %s %d/%d %d #%d:%s %d %s" % \ (si_s, k, N, datalen, seqnum, unicode(base32.b2a(root_hash), "utf-8"), expiration, quote_output(abs_sharefile)), file=out) else: print("UNKNOWN mutable %s" % quote_output(abs_sharefile), file=out) elif struct.unpack(">L", prefix[:4]) == (1, ): # immutable class ImmediateReadBucketProxy(ReadBucketProxy): def __init__(self, sf): self.sf = sf ReadBucketProxy.__init__(self, None, None, "") def __repr__(self): return "<ImmediateReadBucketProxy>" def _read(self, offset, size): return defer.succeed(sf.read_share_data(offset, size)) # use a ReadBucketProxy to parse the bucket and find the uri extension sf = ShareFile(abs_sharefile) bp = ImmediateReadBucketProxy(sf) expiration_time = min( [lease.expiration_time for lease in sf.get_leases()]) expiration = max(0, expiration_time - now) UEB_data = call(bp.get_uri_extension) unpacked = uri.unpack_extension_readable(UEB_data) k = unpacked["needed_shares"] N = unpacked["total_shares"] filesize = unpacked["size"] ueb_hash = unpacked["UEB_hash"] print("CHK %s %d/%d %d %s %d %s" % (si_s, k, N, filesize, unicode( ueb_hash, "utf-8"), expiration, quote_output(abs_sharefile)), file=out) else: print("UNKNOWN really-unknown %s" % quote_output(abs_sharefile), file=out) f.close()
class ImmediateReadBucketProxy(ReadBucketProxy): def __init__(self, sf): self.sf = sf ReadBucketProxy.__init__(self, "", "", "") def __repr__(self): return "<ImmediateReadBucketProxy>" def _read(self, offset, size): return defer.succeed(sf.read_share_data(offset, size)) # use a ReadBucketProxy to parse the bucket and find the uri extension sf = ShareFile(abs_sharefile) bp = ImmediateReadBucketProxy(sf) expiration_time = min( [lease.expiration_time for lease in sf.get_leases()] ) expiration = max(0, expiration_time - now) UEB_data = call(bp.get_uri_extension) unpacked = uri.unpack_extension_readable(UEB_data) k = unpacked["needed_shares"] N = unpacked["total_shares"] filesize = unpacked["size"] ueb_hash = unpacked["UEB_hash"] print >>out, "CHK %s %d/%d %d %s %d %s" % (si_s, k, N, filesize, ueb_hash, expiration, quote_output(abs_sharefile)) else:
class ImmediateReadBucketProxy(ReadBucketProxy): def __init__(self, sf): self.sf = sf ReadBucketProxy.__init__(self, None, None, "") def __repr__(self): return "<ImmediateReadBucketProxy>" def _read(self, offset, size): return defer.succeed(sf.read_share_data(offset, size)) # use a ReadBucketProxy to parse the bucket and find the uri extension sf = ShareFile(abs_sharefile) bp = ImmediateReadBucketProxy(sf) expiration_time = min( [lease.expiration_time for lease in sf.get_leases()] ) expiration = max(0, expiration_time - now) UEB_data = call(bp.get_uri_extension) unpacked = uri.unpack_extension_readable(UEB_data) k = unpacked["needed_shares"] N = unpacked["total_shares"] filesize = unpacked["size"] ueb_hash = unpacked["UEB_hash"] print >>out, "CHK %s %d/%d %d %s %d %s" % (si_s, k, N, filesize, ueb_hash, expiration, quote_output(abs_sharefile)) else:
def __init__(self, sf): self.sf = sf ReadBucketProxy.__init__(self, None, None, "") def __repr__(self): return "<ImmediateReadBucketProxy>" def _read(self, offset, size): return defer.succeed(sf.read_share_data(offset, size)) # use a ReadBucketProxy to parse the bucket and find the uri extension sf = ShareFile(abs_sharefile) bp = ImmediateReadBucketProxy(sf) expiration_time = min( [lease.expiration_time for lease in sf.get_leases()]) expiration = max(0, expiration_time - now) UEB_data = call(bp.get_uri_extension) unpacked = uri.unpack_extension_readable(UEB_data) k = unpacked["needed_shares"] N = unpacked["total_shares"] filesize = unpacked["size"] ueb_hash = unpacked["UEB_hash"] print >> out, "CHK %s %d/%d %d %s %d %s" % ( si_s, k, N, filesize, ueb_hash, expiration, quote_output(abs_sharefile)) else:
def describe_share(abs_sharefile, si_s, shnum_s, now, out): from allmydata import uri from allmydata.storage.mutable import MutableShareFile from allmydata.storage.immutable import ShareFile from allmydata.mutable.layout import unpack_share from allmydata.mutable.common import NeedMoreDataError from allmydata.immutable.layout import ReadBucketProxy from allmydata.util import base32 from allmydata.util.encodingutil import quote_output import struct f = open(abs_sharefile, "rb") prefix = f.read(32) if prefix == MutableShareFile.MAGIC: # mutable share m = MutableShareFile(abs_sharefile) WE, nodeid = m._read_write_enabler_and_nodeid(f) data_length = m._read_data_length(f) expiration_time = min( [lease.expiration_time for (i,lease) in m._enumerate_leases(f)] ) expiration = max(0, expiration_time - now) share_type = "unknown" f.seek(m.DATA_OFFSET) version = f.read(1) if version == "\x00": # this slot contains an SMDF share share_type = "SDMF" elif version == "\x01": share_type = "MDMF" if share_type == "SDMF": f.seek(m.DATA_OFFSET) data = f.read(min(data_length, 2000)) try: pieces = unpack_share(data) except NeedMoreDataError as e: # retry once with the larger size size = e.needed_bytes f.seek(m.DATA_OFFSET) data = f.read(min(data_length, size)) pieces = unpack_share(data) (seqnum, root_hash, IV, k, N, segsize, datalen, pubkey, signature, share_hash_chain, block_hash_tree, share_data, enc_privkey) = pieces print("SDMF %s %d/%d %d #%d:%s %d %s" % \ (si_s, k, N, datalen, seqnum, base32.b2a(root_hash), expiration, quote_output(abs_sharefile)), file=out) elif share_type == "MDMF": from allmydata.mutable.layout import MDMFSlotReadProxy fake_shnum = 0 # TODO: factor this out with dump_MDMF_share() class ShareDumper(MDMFSlotReadProxy): def _read(self, readvs, force_remote=False, queue=False): data = [] for (where,length) in readvs: f.seek(m.DATA_OFFSET+where) data.append(f.read(length)) return defer.succeed({fake_shnum: data}) p = ShareDumper(None, "fake-si", fake_shnum) def extract(func): stash = [] # these methods return Deferreds, but we happen to know that # they run synchronously when not actually talking to a # remote server d = func() d.addCallback(stash.append) return stash[0] verinfo = extract(p.get_verinfo) (seqnum, root_hash, salt_to_use, segsize, datalen, k, N, prefix, offsets) = verinfo print("MDMF %s %d/%d %d #%d:%s %d %s" % \ (si_s, k, N, datalen, seqnum, base32.b2a(root_hash), expiration, quote_output(abs_sharefile)), file=out) else: print("UNKNOWN mutable %s" % quote_output(abs_sharefile), file=out) elif struct.unpack(">L", prefix[:4]) == (1,): # immutable class ImmediateReadBucketProxy(ReadBucketProxy): def __init__(self, sf): self.sf = sf ReadBucketProxy.__init__(self, None, None, "") def __repr__(self): return "<ImmediateReadBucketProxy>" def _read(self, offset, size): return defer.succeed(sf.read_share_data(offset, size)) # use a ReadBucketProxy to parse the bucket and find the uri extension sf = ShareFile(abs_sharefile) bp = ImmediateReadBucketProxy(sf) expiration_time = min( [lease.expiration_time for lease in sf.get_leases()] ) expiration = max(0, expiration_time - now) UEB_data = call(bp.get_uri_extension) unpacked = uri.unpack_extension_readable(UEB_data) k = unpacked["needed_shares"] N = unpacked["total_shares"] filesize = unpacked["size"] ueb_hash = unpacked["UEB_hash"] print("CHK %s %d/%d %d %s %d %s" % (si_s, k, N, filesize, ueb_hash, expiration, quote_output(abs_sharefile)), file=out) else: print("UNKNOWN really-unknown %s" % quote_output(abs_sharefile), file=out) f.close()