def test_client_v2_signed(self): introducer = IntroducerService() tub = introducer_furl = None app_versions = {"whizzy": "fizzy"} client_v2 = IntroducerClient(tub, introducer_furl, u"nick-v2", "my_version", "oldest", app_versions, fakeseq, FilePath(self.mktemp())) furl1 = "pb://[email protected]:0/swissnum" private_key, public_key = ed25519.create_signing_keypair() public_key_str = remove_prefix( ed25519.string_from_verifying_key(public_key), "pub-") ann_t0 = make_ann_t(client_v2, furl1, private_key, 10) canary0 = Referenceable() introducer.remote_publish_v2(ann_t0, canary0) a = introducer.get_announcements() self.failUnlessEqual(len(a), 1) self.assertThat(a[0].canary, Is(canary0)) self.failUnlessEqual(a[0].index, ("storage", public_key_str)) self.failUnlessEqual(a[0].announcement["app-versions"], app_versions) self.failUnlessEqual(a[0].nickname, u"nick-v2") self.failUnlessEqual(a[0].service_name, "storage") self.failUnlessEqual(a[0].version, "my_version") self.failUnlessEqual(a[0].announcement["anonymous-storage-FURL"], furl1)
def test_remove_prefix_entire_string(self): """ removing a prefix which is the whole string is empty """ self.assertEquals( remove_prefix(b"foobar", b"foobar"), b"", )
def test_remove_prefix_zero(self): """ removing a zero-length prefix does nothing """ self.assertEquals( remove_prefix(b"foobar", b""), b"foobar", )
def verifying_key_from_string(public_key_bytes): """ Load a verifying key from a string of bytes (which includes the PUBLIC_KEY_PREFIX) :returns: a public_key """ if not isinstance(public_key_bytes, six.binary_type): raise ValueError('public_key_bytes must be bytes') return Ed25519PublicKey.from_public_bytes( a2b(remove_prefix(public_key_bytes, PUBLIC_KEY_PREFIX)))
def signing_keypair_from_string(private_key_bytes): """ Load a signing keypair from a string of bytes (which includes the PRIVATE_KEY_PREFIX) :returns: a 2-tuple of (private_key, public_key) """ if not isinstance(private_key_bytes, six.binary_type): raise ValueError('private_key_bytes must be bytes') private_key = Ed25519PrivateKey.from_private_bytes( a2b(remove_prefix(private_key_bytes, PRIVATE_KEY_PREFIX))) return private_key, private_key.public_key()
def unsign_from_foolscap(ann_t): (msg, sig_vs, claimed_key_vs) = ann_t if not sig_vs or not claimed_key_vs: raise UnknownKeyError("only signed announcements recognized") if not sig_vs.startswith(b"v0-"): raise UnknownKeyError("only v0- signatures recognized") if not claimed_key_vs.startswith(b"v0-"): raise UnknownKeyError("only v0- keys recognized") claimed_key = ed25519.verifying_key_from_string(b"pub-" + claimed_key_vs) sig_bytes = base32.a2b(remove_prefix(sig_vs, b"v0-")) ed25519.verify_signature(claimed_key, sig_bytes, msg) key_vs = claimed_key_vs ann = json.loads(msg.decode("utf-8")) return (ann, key_vs)
def sign_to_foolscap(announcement, signing_key): """ :param signing_key: a (private) signing key, as returned from e.g. :func:`allmydata.crypto.ed25519.signing_keypair_from_string` :returns: 3-tuple of (msg, sig, vk) where msg is a UTF8 JSON serialization of the `announcement` (bytes), sig is bytes (a signature of msg) and vk is the verifying key bytes """ # return (bytes, sig-str, pubkey-str). A future HTTP-based serialization # will use JSON({msg:b64(JSON(msg).utf8), sig:v0-b64(sig), # pubkey:v0-b64(pubkey)}) . msg = json.dumps(announcement).encode("utf-8") sig = b"v0-" + base32.b2a(ed25519.sign_data(signing_key, msg)) verifying_key_string = ed25519.string_from_verifying_key( ed25519.verifying_key_from_signing_key(signing_key)) ann_t = (msg, sig, remove_prefix(verifying_key_string, b"pub-")) return ann_t
def _init_permutation_seed(self, ss): seed = self.config.get_config_from_file("permutation-seed") if not seed: have_shares = ss.have_shares() if have_shares: # if the server has shares but not a recorded # permutation-seed, then it has been around since pre-#466 # days, and the clients who uploaded those shares used our # TubID as a permutation-seed. We should keep using that same # seed to keep the shares in the same place in the permuted # ring, so those clients don't have to perform excessive # searches. seed = base32.b2a(self.nodeid) else: # otherwise, we're free to use the more natural seed of our # pubkey-based serverid vk_string = ed25519.string_from_verifying_key(self._node_public_key) vk_bytes = remove_prefix(vk_string, ed25519.PUBLIC_KEY_PREFIX) seed = base32.b2a(vk_bytes) self.config.write_config_file("permutation-seed", seed+b"\n", mode="wb") return seed.strip()
def test_client_cache(self): basedir = "introducer/ClientSeqnums/test_client_cache_1" fileutil.make_dirs(basedir) cache_filepath = FilePath( os.path.join(basedir, "private", "introducer_default_cache.yaml")) # if storage is enabled, the Client will publish its storage server # during startup (although the announcement will wait in a queue # until the introducer connection is established). To avoid getting # confused by this, disable storage. with open(os.path.join(basedir, "tahoe.cfg"), "w") as f: f.write("[client]\n") f.write("introducer.furl = nope\n") f.write("[storage]\n") f.write("enabled = false\n") c = yield create_client(basedir) ic = c.introducer_clients[0] private_key, public_key = ed25519.create_signing_keypair() public_key_str = remove_prefix( ed25519.string_from_verifying_key(public_key), "pub-") furl1 = "pb://[email protected]:123/short" # base32("short") ann_t = make_ann_t(ic, furl1, private_key, 1) ic.got_announcements([ann_t]) yield flushEventualQueue() # check the cache for the announcement announcements = self._load_cache(cache_filepath) self.failUnlessEqual(len(announcements), 1) self.failUnlessEqual(announcements[0]['key_s'], public_key_str) ann = announcements[0]["ann"] self.failUnlessEqual(ann["anonymous-storage-FURL"], furl1) self.failUnlessEqual(ann["seqnum"], 1) # a new announcement that replaces the first should replace the # cached entry, not duplicate it furl2 = furl1 + "er" ann_t2 = make_ann_t(ic, furl2, private_key, 2) ic.got_announcements([ann_t2]) yield flushEventualQueue() announcements = self._load_cache(cache_filepath) self.failUnlessEqual(len(announcements), 1) self.failUnlessEqual(announcements[0]['key_s'], public_key_str) ann = announcements[0]["ann"] self.failUnlessEqual(ann["anonymous-storage-FURL"], furl2) self.failUnlessEqual(ann["seqnum"], 2) # but a third announcement with a different key should add to the # cache private_key2, public_key2 = ed25519.create_signing_keypair() public_key_str2 = remove_prefix( ed25519.string_from_verifying_key(public_key2), "pub-") furl3 = "pb://[email protected]:456/short" ann_t3 = make_ann_t(ic, furl3, private_key2, 1) ic.got_announcements([ann_t3]) yield flushEventualQueue() announcements = self._load_cache(cache_filepath) self.failUnlessEqual(len(announcements), 2) self.failUnlessEqual(set([public_key_str, public_key_str2]), set([a["key_s"] for a in announcements])) self.failUnlessEqual( set([furl2, furl3]), set([a["ann"]["anonymous-storage-FURL"] for a in announcements])) # test loading yield flushEventualQueue() ic2 = IntroducerClient(None, "introducer.furl", u"my_nickname", "my_version", "oldest_version", {}, fakeseq, ic._cache_filepath) announcements = {} def got(key_s, ann): announcements[key_s] = ann ic2.subscribe_to("storage", got) ic2._load_announcements() # normally happens when connection fails yield flushEventualQueue() self.failUnless(public_key_str in announcements) self.failUnlessEqual( announcements[public_key_str]["anonymous-storage-FURL"], furl2) self.failUnlessEqual( announcements[public_key_str2]["anonymous-storage-FURL"], furl3) c2 = yield create_client(basedir) c2.introducer_clients[0]._load_announcements() yield flushEventualQueue() self.assertEqual(c2.storage_broker.get_all_serverids(), frozenset([public_key_str, public_key_str2]))
def do_system_test(self): self.create_tub() introducer = IntroducerService() introducer.setServiceParent(self.parent) iff = os.path.join(self.basedir, "introducer.furl") tub = self.central_tub ifurl = self.central_tub.registerReference(introducer, furlFile=iff) self.introducer_furl = ifurl # we have 5 clients who publish themselves as storage servers, and a # sixth which does which not. All 6 clients subscriber to hear about # storage. When the connections are fully established, all six nodes # should have 5 connections each. NUM_STORAGE = 5 NUM_CLIENTS = 6 clients = [] tubs = {} received_announcements = {} subscribing_clients = [] publishing_clients = [] printable_serverids = {} self.the_introducer = introducer privkeys = {} pubkeys = {} expected_announcements = [0 for c in range(NUM_CLIENTS)] for i in range(NUM_CLIENTS): tub = Tub() #tub.setOption("logLocalFailures", True) #tub.setOption("logRemoteFailures", True) tub.setOption("expose-remote-exception-types", False) tub.setServiceParent(self.parent) listenOnUnused(tub) log.msg("creating client %d: %s" % (i, tub.getShortTubID())) c = IntroducerClient(tub, self.introducer_furl, NICKNAME % str(i), "version", "oldest", {"component": "component-v1"}, fakeseq, FilePath(self.mktemp())) received_announcements[c] = {} def got(key_s_or_tubid, ann, announcements): index = key_s_or_tubid or get_tubid_string_from_ann(ann) announcements[index] = ann c.subscribe_to("storage", got, received_announcements[c]) subscribing_clients.append(c) expected_announcements[ i] += 1 # all expect a 'storage' announcement node_furl = tub.registerReference(Referenceable()) private_key, public_key = ed25519.create_signing_keypair() public_key_str = ed25519.string_from_verifying_key(public_key) privkeys[i] = private_key pubkeys[i] = public_key_str if i < NUM_STORAGE: # sign all announcements c.publish("storage", make_ann(node_furl), private_key) printable_serverids[i] = remove_prefix(public_key_str, b"pub-") publishing_clients.append(c) else: # the last one does not publish anything pass if i == 2: # also publish something that nobody cares about boring_furl = tub.registerReference(Referenceable()) c.publish("boring", make_ann(boring_furl), private_key) c.setServiceParent(self.parent) clients.append(c) tubs[c] = tub def _wait_for_connected(ign): def _connected(): for c in clients: if not c.connected_to_introducer(): return False return True return self.poll(_connected) # we watch the clients to determine when the system has settled down. # Then we can look inside the server to assert things about its # state. def _wait_for_expected_announcements(ign): def _got_expected_announcements(): for i, c in enumerate(subscribing_clients): if len(received_announcements[c] ) < expected_announcements[i]: return False return True return self.poll(_got_expected_announcements) # before shutting down any Tub, we'd like to know that there are no # messages outstanding def _wait_until_idle(ign): def _idle(): for c in subscribing_clients + publishing_clients: if c._debug_outstanding: return False if self.the_introducer._debug_outstanding: return False return True return self.poll(_idle) d = defer.succeed(None) d.addCallback(_wait_for_connected) d.addCallback(_wait_for_expected_announcements) d.addCallback(_wait_until_idle) def _check1(res): log.msg("doing _check1") dc = self.the_introducer._debug_counts # each storage server publishes a record. There is also one # "boring" self.failUnlessEqual(dc["inbound_message"], NUM_STORAGE + 1) self.failUnlessEqual(dc["inbound_duplicate"], 0) self.failUnlessEqual(dc["inbound_update"], 0) self.failUnlessEqual(dc["inbound_subscribe"], NUM_CLIENTS) # the number of outbound messages is tricky.. I think it depends # upon a race between the publish and the subscribe messages. self.failUnless(dc["outbound_message"] > 0) # each client subscribes to "storage", and each server publishes self.failUnlessEqual(dc["outbound_announcements"], NUM_STORAGE * NUM_CLIENTS) for c in subscribing_clients: cdc = c._debug_counts self.failUnless(cdc["inbound_message"]) self.failUnlessEqual(cdc["inbound_announcement"], NUM_STORAGE) self.failUnlessEqual(cdc["wrong_service"], 0) self.failUnlessEqual(cdc["duplicate_announcement"], 0) self.failUnlessEqual(cdc["update"], 0) self.failUnlessEqual(cdc["new_announcement"], NUM_STORAGE) anns = received_announcements[c] self.failUnlessEqual(len(anns), NUM_STORAGE) serverid0 = printable_serverids[0] ann = anns[serverid0] nick = ann["nickname"] self.failUnlessEqual(type(nick), unicode) self.failUnlessEqual(nick, NICKNAME % "0") for c in publishing_clients: cdc = c._debug_counts expected = 1 if c in [ clients[2], # boring ]: expected = 2 self.failUnlessEqual(cdc["outbound_message"], expected) # now check the web status, make sure it renders without error ir = introweb.IntroducerRoot(self.parent) self.parent.nodeid = "NODEID" text = ir.renderSynchronously().decode("utf-8") self.assertIn(NICKNAME % "0", text) # a v2 client self.assertIn(NICKNAME % "1", text) # another v2 client for i in range(NUM_STORAGE): self.assertIn(printable_serverids[i], text, (i, printable_serverids[i], text)) # make sure there isn't a double-base32ed string too self.assertNotIn(idlib.nodeid_b2a(printable_serverids[i]), text, (i, printable_serverids[i], text)) log.msg("_check1 done") d.addCallback(_check1) # force an introducer reconnect, by shutting down the Tub it's using # and starting a new Tub (with the old introducer). Everybody should # reconnect and republish, but the introducer should ignore the # republishes as duplicates. However, because the server doesn't know # what each client does and does not know, it will send them a copy # of the current announcement table anyway. d.addCallback(lambda _ign: log.msg("shutting down introducer's Tub")) d.addCallback(lambda _ign: self.central_tub.disownServiceParent()) def _wait_for_introducer_loss(ign): def _introducer_lost(): for c in clients: if c.connected_to_introducer(): return False return True return self.poll(_introducer_lost) d.addCallback(_wait_for_introducer_loss) def _restart_introducer_tub(_ign): log.msg("restarting introducer's Tub") # reset counters for i in range(NUM_CLIENTS): c = subscribing_clients[i] for k in c._debug_counts: c._debug_counts[k] = 0 for k in self.the_introducer._debug_counts: self.the_introducer._debug_counts[k] = 0 expected_announcements[i] += 1 # new 'storage' for everyone self.create_tub(self.central_portnum) newfurl = self.central_tub.registerReference(self.the_introducer, furlFile=iff) assert newfurl == self.introducer_furl d.addCallback(_restart_introducer_tub) d.addCallback(_wait_for_connected) d.addCallback(_wait_for_expected_announcements) d.addCallback(_wait_until_idle) d.addCallback(lambda _ign: log.msg(" reconnected")) # TODO: publish something while the introducer is offline, then # confirm it gets delivered when the connection is reestablished def _check2(res): log.msg("doing _check2") # assert that the introducer sent out new messages, one per # subscriber dc = self.the_introducer._debug_counts self.failUnlessEqual(dc["outbound_announcements"], NUM_STORAGE * NUM_CLIENTS) self.failUnless(dc["outbound_message"] > 0) self.failUnlessEqual(dc["inbound_subscribe"], NUM_CLIENTS) for c in subscribing_clients: cdc = c._debug_counts self.failUnlessEqual(cdc["inbound_message"], 1) self.failUnlessEqual(cdc["inbound_announcement"], NUM_STORAGE) self.failUnlessEqual(cdc["new_announcement"], 0) self.failUnlessEqual(cdc["wrong_service"], 0) self.failUnlessEqual(cdc["duplicate_announcement"], NUM_STORAGE) d.addCallback(_check2) # Then force an introducer restart, by shutting down the Tub, # destroying the old introducer, and starting a new Tub+Introducer. # Everybody should reconnect and republish, and the (new) introducer # will distribute the new announcements, but the clients should # ignore the republishes as duplicates. d.addCallback(lambda _ign: log.msg("shutting down introducer")) d.addCallback(lambda _ign: self.central_tub.disownServiceParent()) d.addCallback(_wait_for_introducer_loss) d.addCallback(lambda _ign: log.msg("introducer lost")) def _restart_introducer(_ign): log.msg("restarting introducer") self.create_tub(self.central_portnum) # reset counters for i in range(NUM_CLIENTS): c = subscribing_clients[i] for k in c._debug_counts: c._debug_counts[k] = 0 expected_announcements[i] += 1 # new 'storage' for everyone introducer = IntroducerService() self.the_introducer = introducer newfurl = self.central_tub.registerReference(self.the_introducer, furlFile=iff) assert newfurl == self.introducer_furl d.addCallback(_restart_introducer) d.addCallback(_wait_for_connected) d.addCallback(_wait_for_expected_announcements) d.addCallback(_wait_until_idle) def _check3(res): log.msg("doing _check3") dc = self.the_introducer._debug_counts self.failUnlessEqual(dc["outbound_announcements"], NUM_STORAGE * NUM_CLIENTS) self.failUnless(dc["outbound_message"] > 0) self.failUnlessEqual(dc["inbound_subscribe"], NUM_CLIENTS) for c in subscribing_clients: cdc = c._debug_counts self.failUnless(cdc["inbound_message"] > 0) self.failUnlessEqual(cdc["inbound_announcement"], NUM_STORAGE) self.failUnlessEqual(cdc["new_announcement"], 0) self.failUnlessEqual(cdc["wrong_service"], 0) self.failUnlessEqual(cdc["duplicate_announcement"], NUM_STORAGE) d.addCallback(_check3) return d
def test_duplicate_receive_v2(self): ic1 = IntroducerClient(None, "introducer.furl", u"my_nickname", "ver23", "oldest_version", {}, fakeseq, FilePath(self.mktemp())) # we use a second client just to create a different-looking # announcement ic2 = IntroducerClient(None, "introducer.furl", u"my_nickname", "ver24", "oldest_version", {}, fakeseq, FilePath(self.mktemp())) announcements = [] def _received(key_s, ann): announcements.append((key_s, ann)) ic1.subscribe_to("storage", _received) furl1 = "pb://[email protected]:36106/gydnp" furl1a = "pb://[email protected]:7777/gydnp" furl2 = "pb://[email protected]:36106/ttwwoo" private_key, public_key = ed25519.create_signing_keypair() public_key_str = ed25519.string_from_verifying_key(public_key) pubkey_s = remove_prefix(public_key_str, "pub-") # ann1: ic1, furl1 # ann1a: ic1, furl1a (same SturdyRef, different connection hints) # ann1b: ic2, furl1 # ann2: ic2, furl2 self.ann1 = make_ann_t(ic1, furl1, private_key, seqnum=10) self.ann1old = make_ann_t(ic1, furl1, private_key, seqnum=9) self.ann1noseqnum = make_ann_t(ic1, furl1, private_key, seqnum=None) self.ann1b = make_ann_t(ic2, furl1, private_key, seqnum=11) self.ann1a = make_ann_t(ic1, furl1a, private_key, seqnum=12) self.ann2 = make_ann_t(ic2, furl2, private_key, seqnum=13) ic1.remote_announce_v2([self.ann1]) # queues eventual-send d = fireEventually() def _then1(ign): self.failUnlessEqual(len(announcements), 1) key_s, ann = announcements[0] self.failUnlessEqual(key_s, pubkey_s) self.failUnlessEqual(ann["anonymous-storage-FURL"], furl1) self.failUnlessEqual(ann["my-version"], "ver23") d.addCallback(_then1) # now send a duplicate announcement. This should not fire the # subscriber d.addCallback(lambda ign: ic1.remote_announce_v2([self.ann1])) d.addCallback(fireEventually) def _then2(ign): self.failUnlessEqual(len(announcements), 1) d.addCallback(_then2) # an older announcement shouldn't fire the subscriber either d.addCallback(lambda ign: ic1.remote_announce_v2([self.ann1old])) d.addCallback(fireEventually) def _then2a(ign): self.failUnlessEqual(len(announcements), 1) d.addCallback(_then2a) # announcement with no seqnum cannot replace one with-seqnum d.addCallback(lambda ign: ic1.remote_announce_v2([self.ann1noseqnum])) d.addCallback(fireEventually) def _then2b(ign): self.failUnlessEqual(len(announcements), 1) d.addCallback(_then2b) # and a replacement announcement: same FURL, new other stuff. The # subscriber *should* be fired. d.addCallback(lambda ign: ic1.remote_announce_v2([self.ann1b])) d.addCallback(fireEventually) def _then3(ign): self.failUnlessEqual(len(announcements), 2) key_s, ann = announcements[-1] self.failUnlessEqual(key_s, pubkey_s) self.failUnlessEqual(ann["anonymous-storage-FURL"], furl1) self.failUnlessEqual(ann["my-version"], "ver24") d.addCallback(_then3) # and a replacement announcement with a different FURL (it uses # different connection hints) d.addCallback(lambda ign: ic1.remote_announce_v2([self.ann1a])) d.addCallback(fireEventually) def _then4(ign): self.failUnlessEqual(len(announcements), 3) key_s, ann = announcements[-1] self.failUnlessEqual(key_s, pubkey_s) self.failUnlessEqual(ann["anonymous-storage-FURL"], furl1a) self.failUnlessEqual(ann["my-version"], "ver23") d.addCallback(_then4) # now add a new subscription, which should be called with the # backlog. The introducer only records one announcement per index, so # the backlog will only have the latest message. announcements2 = [] def _received2(key_s, ann): announcements2.append((key_s, ann)) d.addCallback(lambda ign: ic1.subscribe_to("storage", _received2)) d.addCallback(fireEventually) def _then5(ign): self.failUnlessEqual(len(announcements2), 1) key_s, ann = announcements2[-1] self.failUnlessEqual(key_s, pubkey_s) self.failUnlessEqual(ann["anonymous-storage-FURL"], furl1a) self.failUnlessEqual(ann["my-version"], "ver23") d.addCallback(_then5) return d
def get_long_nodeid(self): # this matches what IServer.get_longname() says about us elsewhere vk_string = ed25519.string_from_verifying_key(self._node_public_key) return remove_prefix(vk_string, "pub-")
def test_remove_prefix_partial(self): """ removing a prefix with only partial match fails with exception """ with self.assertRaises(BadPrefixError): remove_prefix(b"foobar", b"fooz"),
def test_remove_prefix_bad(self): """ attempt to remove a prefix that doesn't exist fails with exception """ with self.assertRaises(BadPrefixError): remove_prefix(b"foobar", b"bar")
def test_remove_prefix_good(self): """ remove a simple prefix properly """ self.assertEquals(remove_prefix(b"foobar", b"foo"), b"bar")