def test_pack_badly(self): key = "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f" storage_index = hashutil.storage_index_hash(key) uri_extension_hash = hashutil.uri_extension_hash("stuff") needed_shares = 25 total_shares = 100 size = 1234 self.failUnlessRaises(TypeError, uri.CHKFileURI, key=key, uri_extension_hash=uri_extension_hash, needed_shares=needed_shares, total_shares=total_shares, size=size, bogus_extra_argument="reject me", ) self.failUnlessRaises(TypeError, uri.CHKFileVerifierURI, bogus="bogus") self.failUnlessRaises(TypeError, uri.CHKFileVerifierURI, storage_index=storage_index, uri_extension_hash=uri_extension_hash, needed_shares=3, total_shares=10, # leave size= missing )
def test_immutable(self): readkey = "\x01" * 16 uri_extension_hash = hashutil.uri_extension_hash("stuff") needed_shares = 3 total_shares = 10 size = 1234 fnuri = uri.CHKFileURI(key=readkey, uri_extension_hash=uri_extension_hash, needed_shares=needed_shares, total_shares=total_shares, size=size) fncap = fnuri.to_string() self.failUnlessReallyEqual(fncap, "URI:CHK:aeaqcaibaeaqcaibaeaqcaibae:nf3nimquen7aeqm36ekgxomalstenpkvsdmf6fplj7swdatbv5oa:3:10:1234") u1 = uri.ImmutableDirectoryURI(fnuri) self.failUnless(u1.is_readonly()) self.failIf(u1.is_mutable()) self.failUnless(IURI.providedBy(u1)) self.failIf(IFileURI.providedBy(u1)) self.failUnless(IDirnodeURI.providedBy(u1)) self.failUnless("DirectoryURI" in str(u1)) u1_filenode = u1.get_filenode_cap() self.failIf(u1_filenode.is_mutable()) self.failUnless(u1_filenode.is_readonly()) self.failUnlessReallyEqual(u1_filenode.to_string(), fncap) self.failUnless(str(u1)) u2 = uri.from_string(u1.to_string()) self.failUnlessReallyEqual(u1.to_string(), u2.to_string()) self.failUnless(u2.is_readonly()) self.failIf(u2.is_mutable()) self.failUnless(IURI.providedBy(u2)) self.failIf(IFileURI.providedBy(u2)) self.failUnless(IDirnodeURI.providedBy(u2)) u2i = uri.from_string(u1.to_string(), deep_immutable=True) self.failUnlessReallyEqual(u1.to_string(), u2i.to_string()) u3 = u2.get_readonly() self.failUnlessReallyEqual(u3.to_string(), u2.to_string()) self.failUnless(str(u3)) u3i = uri.from_string(u2.to_string(), deep_immutable=True) self.failUnlessReallyEqual(u2.to_string(), u3i.to_string()) u2_verifier = u2.get_verify_cap() self.failUnless(isinstance(u2_verifier, uri.ImmutableDirectoryURIVerifier), u2_verifier) self.failUnless(IVerifierURI.providedBy(u2_verifier)) u2vs = u2_verifier.to_string() # URI:DIR2-CHK-Verifier:$key:$ueb:$k:$n:$size self.failUnless(u2vs.startswith("URI:DIR2-CHK-Verifier:"), u2vs) u2_verifier_fileuri = u2_verifier.get_filenode_cap() self.failUnless(IVerifierURI.providedBy(u2_verifier_fileuri)) u2vfs = u2_verifier_fileuri.to_string() # URI:CHK-Verifier:$key:$ueb:$k:$n:$size self.failUnlessReallyEqual(u2vfs, fnuri.get_verify_cap().to_string()) self.failUnlessReallyEqual(u2vs[len("URI:DIR2-"):], u2vfs[len("URI:"):]) self.failUnless(str(u2_verifier))
def unpack_extension_readable(data): unpacked = unpack_extension(data) unpacked["UEB_hash"] = hashutil.uri_extension_hash(data) for k in sorted(unpacked.keys()): if 'hash' in k: unpacked[k] = base32.b2a(unpacked[k]) return unpacked
def test_pack_badly(self): key = "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f" storage_index = hashutil.storage_index_hash(key) uri_extension_hash = hashutil.uri_extension_hash("stuff") needed_shares = 25 total_shares = 100 size = 1234 self.failUnlessRaises( TypeError, uri.CHKFileURI, key=key, uri_extension_hash=uri_extension_hash, needed_shares=needed_shares, total_shares=total_shares, size=size, bogus_extra_argument="reject me", ) self.failUnlessRaises(TypeError, uri.CHKFileVerifierURI, bogus="bogus") self.failUnlessRaises( TypeError, uri.CHKFileVerifierURI, storage_index=storage_index, uri_extension_hash=uri_extension_hash, needed_shares=3, total_shares=10, # leave size= missing )
def _test(self, uebdict): uebstring = uri.pack_extension(uebdict) uebhash = hashutil.uri_extension_hash(uebstring) fb = FakeBucketReaderWriterProxy() fb.put_uri_extension(uebstring) verifycap = uri.CHKFileVerifierURI(storage_index='x'*16, uri_extension_hash=uebhash, needed_shares=self.K, total_shares=self.M, size=self.SIZE) vup = checker.ValidatedExtendedURIProxy(fb, verifycap) return vup.start()
def _check_integrity(self, data): h = uri_extension_hash(data) if h != self._verifycap.uri_extension_hash: msg = ("The copy of uri_extension we received from %s was bad: wanted %r, got %r" % (self._readbucketproxy, base32.b2a(self._verifycap.uri_extension_hash), base32.b2a(h))) if self._fetch_failures is not None: self._fetch_failures["uri_extension"] += 1 raise BadURIExtensionHashValue(msg) else: return data
def _check_integrity(self, data): h = uri_extension_hash(data) if h != self._verifycap.uri_extension_hash: msg = ("The copy of uri_extension we received from %s was bad: wanted %s, got %s" % (self._readbucketproxy, base32.b2a(self._verifycap.uri_extension_hash), base32.b2a(h))) if self._fetch_failures is not None: self._fetch_failures["uri_extension"] += 1 raise BadURIExtensionHashValue(msg) else: return data
def test_pack(self): data = {"stuff": "value", "size": 12, "needed_shares": 3, "big_hash": hashutil.tagged_hash("foo", "bar")} ext = uri.pack_extension(data) d = uri.unpack_extension(ext) self.failUnlessReallyEqual(d["stuff"], "value") self.failUnlessReallyEqual(d["size"], 12) self.failUnlessReallyEqual(d["big_hash"], hashutil.tagged_hash("foo", "bar")) readable = uri.unpack_extension_readable(ext) self.failUnlessReallyEqual(readable["needed_shares"], 3) self.failUnlessReallyEqual(readable["stuff"], "value") self.failUnlessReallyEqual(readable["size"], 12) self.failUnlessReallyEqual(readable["big_hash"], base32.b2a(hashutil.tagged_hash("foo", "bar"))) self.failUnlessReallyEqual(readable["UEB_hash"], base32.b2a(hashutil.uri_extension_hash(ext)))
def validate_and_store_UEB(self, UEB_s): log.msg("validate_and_store_UEB", level=log.OPERATIONAL, parent=self._lp, umid="7sTrPw") h = hashutil.uri_extension_hash(UEB_s) if h != self._verifycap.uri_extension_hash: raise BadHashError self._parse_and_store_UEB(UEB_s) # sets self._stuff # TODO: a malformed (but authentic) UEB could throw an assertion in # _parse_and_store_UEB, and we should abandon the download. self.have_UEB = True # inform the ShareFinder about our correct number of segments. This # will update the block-hash-trees in all existing CommonShare # instances, and will populate new ones with the correct value. self._sharefinder.update_num_segments()
def _check_chk(self, storage_index, lp): res = upload.HelperUploadResults() res.uri_extension_hash = hashutil.uri_extension_hash("") # we're pretending that the file they're trying to upload was already # present in the grid. We return some information about the file, so # the client can decide if they like the way it looks. The parameters # used here are chosen to match the defaults. PARAMS = FakeClient.DEFAULT_ENCODING_PARAMETERS ueb_data = {"needed_shares": PARAMS["k"], "total_shares": PARAMS["n"], "segment_size": min(PARAMS["max_segment_size"], len(DATA)), "size": len(DATA), } res.uri_extension_data = ueb_data return defer.succeed(res)
def start(self): res = upload.UploadResults() res.uri_extension_hash = hashutil.uri_extension_hash("") # we're pretending that the file they're trying to upload was already # present in the grid. We return some information about the file, so # the client can decide if they like the way it looks. The parameters # used here are chosen to match the defaults. PARAMS = FakeClient.DEFAULT_ENCODING_PARAMETERS ueb_data = {"needed_shares": PARAMS["k"], "total_shares": PARAMS["n"], "segment_size": min(PARAMS["max_segment_size"], len(DATA)), "size": len(DATA), } res.uri_extension_data = ueb_data return (res, None)
def _check_chk(self, storage_index, lp): res = upload.HelperUploadResults() res.uri_extension_hash = hashutil.uri_extension_hash("") # we're pretending that the file they're trying to upload was already # present in the grid. We return some information about the file, so # the client can decide if they like the way it looks. The parameters # used here are chosen to match the defaults. PARAMS = FakeClient.encoding_params ueb_data = { "needed_shares": PARAMS["k"], "total_shares": PARAMS["n"], "segment_size": min(PARAMS["max_segment_size"], len(DATA)), "size": len(DATA), } res.uri_extension_data = ueb_data return defer.succeed(res)
def send_uri_extension_to_all_shareholders(self): lp = self.log("sending uri_extension", level=log.NOISY) self.set_status("Sending URI Extensions") self.set_encode_and_push_progress(extra=0.8) for k in ('crypttext_root_hash', 'crypttext_hash', ): assert k in self.uri_extension_data uri_extension = uri.pack_extension(self.uri_extension_data) ed = {} for k,v in self.uri_extension_data.items(): if k.endswith("hash"): ed[k] = base32.b2a(v) else: ed[k] = v self.log("uri_extension_data is %s" % (ed,), level=log.NOISY, parent=lp) self.uri_extension_hash = hashutil.uri_extension_hash(uri_extension) dl = [] for shareid in list(self.landlords): dl.append(self.send_uri_extension(shareid, uri_extension)) return self._gather_responses(dl)
def test_pack(self): data = {"stuff": "value", "size": 12, "needed_shares": 3, "big_hash": hashutil.tagged_hash("foo", "bar"), } ext = uri.pack_extension(data) d = uri.unpack_extension(ext) self.failUnlessReallyEqual(d["stuff"], "value") self.failUnlessReallyEqual(d["size"], 12) self.failUnlessReallyEqual(d["big_hash"], hashutil.tagged_hash("foo", "bar")) readable = uri.unpack_extension_readable(ext) self.failUnlessReallyEqual(readable["needed_shares"], 3) self.failUnlessReallyEqual(readable["stuff"], "value") self.failUnlessReallyEqual(readable["size"], 12) self.failUnlessReallyEqual(readable["big_hash"], base32.b2a(hashutil.tagged_hash("foo", "bar"))) self.failUnlessReallyEqual(readable["UEB_hash"], base32.b2a(hashutil.uri_extension_hash(ext)))
def test_hashers(self): h1 = hashutil.block_hash(b"foo") h2 = hashutil.block_hasher() h2.update(b"foo") self.failUnlessEqual(h1, h2.digest()) self.assertIsInstance(h1, bytes) h1 = hashutil.uri_extension_hash(b"foo") h2 = hashutil.uri_extension_hasher() h2.update(b"foo") self.failUnlessEqual(h1, h2.digest()) self.assertIsInstance(h1, bytes) h1 = hashutil.plaintext_hash(b"foo") h2 = hashutil.plaintext_hasher() h2.update(b"foo") self.failUnlessEqual(h1, h2.digest()) self.assertIsInstance(h1, bytes) h1 = hashutil.crypttext_hash(b"foo") h2 = hashutil.crypttext_hasher() h2.update(b"foo") self.failUnlessEqual(h1, h2.digest()) self.assertIsInstance(h1, bytes) h1 = hashutil.crypttext_segment_hash(b"foo") h2 = hashutil.crypttext_segment_hasher() h2.update(b"foo") self.failUnlessEqual(h1, h2.digest()) self.assertIsInstance(h1, bytes) h1 = hashutil.plaintext_segment_hash(b"foo") h2 = hashutil.plaintext_segment_hasher() h2.update(b"foo") self.failUnlessEqual(h1, h2.digest()) self.assertIsInstance(h1, bytes)
def _ueb_hash(self): return hashutil.uri_extension_hash(uri.pack_extension( self._ueb_data), )
def test_pack(self): key = "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f" storage_index = hashutil.storage_index_hash(key) uri_extension_hash = hashutil.uri_extension_hash("stuff") needed_shares = 25 total_shares = 100 size = 1234 u = uri.CHKFileURI(key=key, uri_extension_hash=uri_extension_hash, needed_shares=needed_shares, total_shares=total_shares, size=size) self.failUnlessReallyEqual(u.get_storage_index(), storage_index) self.failUnlessReallyEqual(u.key, key) self.failUnlessReallyEqual(u.uri_extension_hash, uri_extension_hash) self.failUnlessReallyEqual(u.needed_shares, needed_shares) self.failUnlessReallyEqual(u.total_shares, total_shares) self.failUnlessReallyEqual(u.size, size) self.failUnless(u.is_readonly()) self.failIf(u.is_mutable()) self.failUnless(IURI.providedBy(u)) self.failUnless(IFileURI.providedBy(u)) self.failIf(IDirnodeURI.providedBy(u)) self.failUnlessReallyEqual(u.get_size(), 1234) u_ro = u.get_readonly() self.failUnlessIdentical(u, u_ro) he = u.to_human_encoding() self.failUnlessReallyEqual(he, "http://127.0.0.1:3456/uri/" + u.to_string()) self.failUnlessReallyEqual(uri.CHKFileURI.init_from_human_encoding(he), u) u2 = uri.from_string(u.to_string()) self.failUnlessReallyEqual(u2.get_storage_index(), storage_index) self.failUnlessReallyEqual(u2.key, key) self.failUnlessReallyEqual(u2.uri_extension_hash, uri_extension_hash) self.failUnlessReallyEqual(u2.needed_shares, needed_shares) self.failUnlessReallyEqual(u2.total_shares, total_shares) self.failUnlessReallyEqual(u2.size, size) self.failUnless(u2.is_readonly()) self.failIf(u2.is_mutable()) self.failUnless(IURI.providedBy(u2)) self.failUnless(IFileURI.providedBy(u2)) self.failIf(IDirnodeURI.providedBy(u2)) self.failUnlessReallyEqual(u2.get_size(), 1234) u2i = uri.from_string(u.to_string(), deep_immutable=True) self.failUnlessReallyEqual(u.to_string(), u2i.to_string()) u2ro = uri.from_string(uri.ALLEGED_READONLY_PREFIX + u.to_string()) self.failUnlessReallyEqual(u.to_string(), u2ro.to_string()) u2imm = uri.from_string(uri.ALLEGED_IMMUTABLE_PREFIX + u.to_string()) self.failUnlessReallyEqual(u.to_string(), u2imm.to_string()) v = u.get_verify_cap() self.failUnless(isinstance(v.to_string(), str)) self.failUnless(v.is_readonly()) self.failIf(v.is_mutable()) v2 = uri.from_string(v.to_string()) self.failUnlessReallyEqual(v, v2) he = v.to_human_encoding() v2_h = uri.CHKFileVerifierURI.init_from_human_encoding(he) self.failUnlessReallyEqual(v2, v2_h) v3 = uri.CHKFileVerifierURI(storage_index="\x00"*16, uri_extension_hash="\x00"*32, needed_shares=3, total_shares=10, size=1234) self.failUnless(isinstance(v3.to_string(), str)) self.failUnless(v3.is_readonly()) self.failIf(v3.is_mutable())
def test_pack(self): key = "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f" storage_index = hashutil.storage_index_hash(key) uri_extension_hash = hashutil.uri_extension_hash("stuff") needed_shares = 25 total_shares = 100 size = 1234 u = uri.CHKFileURI(key=key, uri_extension_hash=uri_extension_hash, needed_shares=needed_shares, total_shares=total_shares, size=size) self.failUnlessReallyEqual(u.get_storage_index(), storage_index) self.failUnlessReallyEqual(u.key, key) self.failUnlessReallyEqual(u.uri_extension_hash, uri_extension_hash) self.failUnlessReallyEqual(u.needed_shares, needed_shares) self.failUnlessReallyEqual(u.total_shares, total_shares) self.failUnlessReallyEqual(u.size, size) self.failUnless(u.is_readonly()) self.failIf(u.is_mutable()) self.failUnless(IURI.providedBy(u)) self.failUnless(IFileURI.providedBy(u)) self.failIf(IDirnodeURI.providedBy(u)) self.failUnlessReallyEqual(u.get_size(), 1234) u_ro = u.get_readonly() self.failUnlessIdentical(u, u_ro) u2 = uri.from_string(u.to_string()) self.failUnlessReallyEqual(u2.get_storage_index(), storage_index) self.failUnlessReallyEqual(u2.key, key) self.failUnlessReallyEqual(u2.uri_extension_hash, uri_extension_hash) self.failUnlessReallyEqual(u2.needed_shares, needed_shares) self.failUnlessReallyEqual(u2.total_shares, total_shares) self.failUnlessReallyEqual(u2.size, size) self.failUnless(u2.is_readonly()) self.failIf(u2.is_mutable()) self.failUnless(IURI.providedBy(u2)) self.failUnless(IFileURI.providedBy(u2)) self.failIf(IDirnodeURI.providedBy(u2)) self.failUnlessReallyEqual(u2.get_size(), 1234) u2i = uri.from_string(u.to_string(), deep_immutable=True) self.failUnlessReallyEqual(u.to_string(), u2i.to_string()) u2ro = uri.from_string(uri.ALLEGED_READONLY_PREFIX + u.to_string()) self.failUnlessReallyEqual(u.to_string(), u2ro.to_string()) u2imm = uri.from_string(uri.ALLEGED_IMMUTABLE_PREFIX + u.to_string()) self.failUnlessReallyEqual(u.to_string(), u2imm.to_string()) v = u.get_verify_cap() self.failUnless(isinstance(v.to_string(), str)) self.failUnless(v.is_readonly()) self.failIf(v.is_mutable()) v2 = uri.from_string(v.to_string()) self.failUnlessReallyEqual(v, v2) v3 = uri.CHKFileVerifierURI(storage_index="\x00" * 16, uri_extension_hash="\x00" * 32, needed_shares=3, total_shares=10, size=1234) self.failUnless(isinstance(v3.to_string(), str)) self.failUnless(v3.is_readonly()) self.failIf(v3.is_mutable())
def _got_uri_extension(self, ueb): self.log("_got_uri_extension", level=log.NOISY) self._ueb_hash = hashutil.uri_extension_hash(ueb) self._ueb_data = uri.unpack_extension(ueb)