def test_client_v2_signed(self): introducer = IntroducerService() tub = introducer_furl = None app_versions = {"whizzy": "fizzy"} client_v2 = IntroducerClient(tub, introducer_furl, u"nick-v2", "my_version", "oldest", app_versions, fakeseq, FilePath(self.mktemp())) furl1 = "pb://[email protected]:0/swissnum" private_key, public_key = ed25519.create_signing_keypair() public_key_str = remove_prefix( ed25519.string_from_verifying_key(public_key), "pub-") ann_t0 = make_ann_t(client_v2, furl1, private_key, 10) canary0 = Referenceable() introducer.remote_publish_v2(ann_t0, canary0) a = introducer.get_announcements() self.failUnlessEqual(len(a), 1) self.assertThat(a[0].canary, Is(canary0)) self.failUnlessEqual(a[0].index, ("storage", public_key_str)) self.failUnlessEqual(a[0].announcement["app-versions"], app_versions) self.failUnlessEqual(a[0].nickname, u"nick-v2") self.failUnlessEqual(a[0].service_name, "storage") self.failUnlessEqual(a[0].version, "my_version") self.failUnlessEqual(a[0].announcement["anonymous-storage-FURL"], furl1)
def test_queue_until_connected(self): self.basedir = "introducer/QueueUntilConnected/queued" os.makedirs(self.basedir) self.create_tub() introducer = IntroducerService() introducer.setServiceParent(self.parent) iff = os.path.join(self.basedir, "introducer.furl") ifurl = self.central_tub.registerReference(introducer, furlFile=iff) tub2 = Tub() tub2.setServiceParent(self.parent) c = IntroducerClient(tub2, ifurl, u"nickname", "version", "oldest", {}, fakeseq, FilePath(self.mktemp())) furl1 = "pb://[email protected]:123/short" # base32("short") sk_s, vk_s = keyutil.make_keypair() sk, _ignored = keyutil.parse_privkey(sk_s) d = introducer.disownServiceParent() def _offline(ign): # now that the introducer server is offline, create a client and # publish some messages c.setServiceParent(self.parent) # this starts the reconnector c.publish("storage", make_ann(furl1), sk) introducer.setServiceParent(self.parent) # restart the server # now wait for the messages to be delivered def _got_announcement(): return bool(introducer.get_announcements()) return self.poll(_got_announcement) d.addCallback(_offline) def _done(ign): v = introducer.get_announcements()[0] furl = v.announcement["anonymous-storage-FURL"] self.failUnlessEqual(furl, furl1) d.addCallback(_done) # now let the ack get back def _wait_until_idle(ign): def _idle(): if c._debug_outstanding: return False if introducer._debug_outstanding: return False return True return self.poll(_idle) d.addCallback(_wait_until_idle) return d
def test_id_collision(self): # test replacement case where tubid equals a keyid (one should # not replace the other) i = IntroducerService() ic = IntroducerClient(None, "introducer.furl", u"my_nickname", "my_version", "oldest_version", {}) sk_s, vk_s = keyutil.make_keypair() sk, _ignored = keyutil.parse_privkey(sk_s) keyid = keyutil.remove_prefix(vk_s, "pub-v0-") furl1 = "pb://[email protected]:123/short" # base32("short") ann_t = ic.create_announcement("storage", make_ann(furl1), sk) i.remote_publish_v2(ann_t, Referenceable()) announcements = i.get_announcements() self.failUnlessEqual(len(announcements), 1) key1 = ("storage", "v0-" + keyid, None) self.failUnlessEqual(announcements[0].index, key1) ann1_out = announcements[0].announcement self.failUnlessEqual(ann1_out["anonymous-storage-FURL"], furl1) furl2 = "pb://%[email protected]:36106/swissnum" % keyid ann2 = (furl2, "storage", "RIStorage", "nick1", "ver23", "ver0") i.remote_publish(ann2) announcements = i.get_announcements() self.failUnlessEqual(len(announcements), 2) key2 = ("storage", None, keyid) wanted = [ad for ad in announcements if ad.index == key2] self.failUnlessEqual(len(wanted), 1) ann2_out = wanted[0].announcement self.failUnlessEqual(ann2_out["anonymous-storage-FURL"], furl2)
def test_id_collision(self): # test replacement case where tubid equals a keyid (one should # not replace the other) i = IntroducerService() ic = IntroducerClient(None, "introducer.furl", u"my_nickname", "my_version", "oldest_version", {}) sk_s, vk_s = keyutil.make_keypair() sk, _ignored = keyutil.parse_privkey(sk_s) keyid = keyutil.remove_prefix(vk_s, "pub-v0-") furl1 = "pb://[email protected]:123/short" # base32("short") ann_t = ic.create_announcement("storage", make_ann(furl1), sk) i.remote_publish_v2(ann_t, Referenceable()) announcements = i.get_announcements() self.failUnlessEqual(len(announcements), 1) key1 = ("storage", "v0-"+keyid, None) self.failUnlessEqual(announcements[0].index, key1) ann1_out = announcements[0].announcement self.failUnlessEqual(ann1_out["anonymous-storage-FURL"], furl1) furl2 = "pb://%[email protected]:36106/swissnum" % keyid ann2 = (furl2, "storage", "RIStorage", "nick1", "ver23", "ver0") i.remote_publish(ann2) announcements = i.get_announcements() self.failUnlessEqual(len(announcements), 2) key2 = ("storage", None, keyid) wanted = [ad for ad in announcements if ad.index == key2] self.failUnlessEqual(len(wanted), 1) ann2_out = wanted[0].announcement self.failUnlessEqual(ann2_out["anonymous-storage-FURL"], furl2)
def test_client_v2(self): introducer = IntroducerService() tub = introducer_furl = None client_v2 = IntroducerClient(tub, introducer_furl, NICKNAME % u"v2", "my_version", "oldest", fakeseq, FilePath(self.mktemp())) #furl1 = "pb://[email protected]:0/swissnum" #ann_s = make_ann_t(client_v2, furl1, None, 10) #introducer.remote_publish_v2(ann_s, Referenceable()) subscriber = FakeRemoteReference() introducer.remote_subscribe_v2(subscriber, "storage", client_v2._my_subscriber_info) subs = introducer.get_subscribers() self.failUnlessEqual(len(subs), 1) s0 = subs[0] self.failUnlessEqual(s0.service_name, "storage") self.failUnlessEqual(s0.nickname, NICKNAME % u"v2") self.failUnlessEqual(s0.version, "my_version")
def test_client_v1(self): introducer = IntroducerService() furl1 = "pb://[email protected]:0/swissnum" tubid = "62ubehyunnyhzs7r6vdonnm2hpi52w6y" ann = (furl1, "storage", "RIStorage", u"nick-v1".encode("utf-8"), "my_version", "oldest") introducer.remote_publish(ann) a = introducer.get_announcements() self.failUnlessEqual(len(a), 1) self.failUnlessEqual(a[0].index, ("storage", None, tubid)) self.failUnlessEqual(a[0].canary, None) self.failUnlessEqual(a[0].announcement["app-versions"], {}) self.failUnlessEqual(a[0].nickname, u"nick-v1".encode("utf-8")) self.failUnlessEqual(a[0].service_name, "storage") self.failUnlessEqual(a[0].version, "my_version") self.failUnlessEqual(a[0].announcement["anonymous-storage-FURL"], furl1)
def test_client_v2(self): introducer = IntroducerService() tub = introducer_furl = None app_versions = {"whizzy": "fizzy"} client_v2 = IntroducerClient(tub, introducer_furl, NICKNAME % u"v2", "my_version", "oldest", app_versions) #furl1 = "pb://[email protected]:0/swissnum" #ann_s = make_ann_t(client_v2, furl1, None) #introducer.remote_publish_v2(ann_s, Referenceable()) subscriber = FakeRemoteReference() introducer.remote_subscribe_v2(subscriber, "storage", client_v2._my_subscriber_info) subs = introducer.get_subscribers() self.failUnlessEqual(len(subs), 1) s0 = subs[0] self.failUnlessEqual(s0.service_name, "storage") self.failUnlessEqual(s0.app_versions, app_versions) self.failUnlessEqual(s0.nickname, NICKNAME % u"v2") self.failUnlessEqual(s0.version, "my_version")
def test_client_v2_unsigned(self): introducer = IntroducerService() tub = introducer_furl = None app_versions = {"whizzy": "fizzy"} client_v2 = IntroducerClient(tub, introducer_furl, u"nick-v2", "my_version", "oldest", app_versions) furl1 = "pb://[email protected]:0/swissnum" tubid = "62ubehyunnyhzs7r6vdonnm2hpi52w6y" ann_s0 = make_ann_t(client_v2, furl1, None) canary0 = Referenceable() introducer.remote_publish_v2(ann_s0, canary0) a = introducer.get_announcements() self.failUnlessEqual(len(a), 1) self.failUnlessIdentical(a[0].canary, canary0) self.failUnlessEqual(a[0].index, ("storage", None, tubid)) self.failUnlessEqual(a[0].announcement["app-versions"], app_versions) self.failUnlessEqual(a[0].nickname, u"nick-v2") self.failUnlessEqual(a[0].service_name, "storage") self.failUnlessEqual(a[0].version, "my_version") self.failUnlessEqual(a[0].announcement["anonymous-storage-FURL"], furl1)
def test_client_v2_unsigned(self): introducer = IntroducerService() tub = introducer_furl = None app_versions = {"whizzy": "fizzy"} client_v2 = IntroducerClient(tub, introducer_furl, u"nick-v2", "my_version", "oldest", app_versions) furl1 = "pb://[email protected]:0/swissnum" tubid = "62ubehyunnyhzs7r6vdonnm2hpi52w6y" ann_s0 = make_ann_t(client_v2, furl1, None, 10.0) canary0 = Referenceable() introducer.remote_publish_v2(ann_s0, canary0) a = introducer.get_announcements() self.failUnlessEqual(len(a), 1) self.failUnlessIdentical(a[0].canary, canary0) self.failUnlessEqual(a[0].index, ("storage", None, tubid)) self.failUnlessEqual(a[0].announcement["app-versions"], app_versions) self.failUnlessEqual(a[0].nickname, u"nick-v2") self.failUnlessEqual(a[0].service_name, "storage") self.failUnlessEqual(a[0].version, "my_version") self.failUnlessEqual(a[0].announcement["anonymous-storage-FURL"], furl1)
def test_client_v1(self): introducer = IntroducerService() subscriber = FakeRemoteReference() introducer.remote_subscribe(subscriber, "storage") # the v1 subscribe interface had no subscriber_info: that was usually # sent in a separate stub_client pseudo-announcement subs = introducer.get_subscribers() self.failUnlessEqual(len(subs), 1) s0 = subs[0] self.failUnlessEqual(s0.nickname, u"?") # not known yet self.failUnlessEqual(s0.service_name, "storage") # now submit the stub_client announcement furl1 = "pb://[email protected]:0/swissnum" ann = (furl1, "stub_client", "RIStubClient", (NICKNAME % u"v1").encode("utf-8"), "my_version", "oldest") introducer.remote_publish(ann) # the server should correlate the two subs = introducer.get_subscribers() self.failUnlessEqual(len(subs), 1) s0 = subs[0] self.failUnlessEqual(s0.service_name, "storage") # v1 announcements do not contain app-versions self.failUnlessEqual(s0.app_versions, {}) self.failUnlessEqual(s0.nickname, NICKNAME % u"v1") self.failUnlessEqual(s0.version, "my_version") # a subscription that arrives after the stub_client announcement # should be correlated too subscriber2 = FakeRemoteReference() introducer.remote_subscribe(subscriber2, "thing2") subs = introducer.get_subscribers() self.failUnlessEqual(len(subs), 2) s0 = [s for s in subs if s.service_name == "thing2"][0] # v1 announcements do not contain app-versions self.failUnlessEqual(s0.app_versions, {}) self.failUnlessEqual(s0.nickname, NICKNAME % u"v1") self.failUnlessEqual(s0.version, "my_version")
def test_client_v2_signed(self): introducer = IntroducerService() tub = introducer_furl = None app_versions = {"whizzy": "fizzy"} client_v2 = IntroducerClient(tub, introducer_furl, u"nick-v2", "my_version", "oldest", app_versions) furl1 = "pb://[email protected]:0/swissnum" sk_s, vk_s = keyutil.make_keypair() sk, _ignored = keyutil.parse_privkey(sk_s) pks = keyutil.remove_prefix(vk_s, "pub-") ann_t0 = make_ann_t(client_v2, furl1, sk) canary0 = Referenceable() introducer.remote_publish_v2(ann_t0, canary0) a = introducer.get_announcements() self.failUnlessEqual(len(a), 1) self.failUnlessIdentical(a[0].canary, canary0) self.failUnlessEqual(a[0].index, ("storage", pks, None)) self.failUnlessEqual(a[0].announcement["app-versions"], app_versions) self.failUnlessEqual(a[0].nickname, u"nick-v2") self.failUnlessEqual(a[0].service_name, "storage") self.failUnlessEqual(a[0].version, "my_version") self.failUnlessEqual(a[0].announcement["anonymous-storage-FURL"], furl1)
def test_client_v2_signed(self): introducer = IntroducerService() tub = introducer_furl = None app_versions = {"whizzy": "fizzy"} client_v2 = IntroducerClient(tub, introducer_furl, u"nick-v2", "my_version", "oldest", app_versions, fakeseq, FilePath(self.mktemp())) furl1 = "pb://[email protected]:0/swissnum" sk_s, vk_s = keyutil.make_keypair() sk, _ignored = keyutil.parse_privkey(sk_s) pks = keyutil.remove_prefix(vk_s, "pub-") ann_t0 = make_ann_t(client_v2, furl1, sk, 10) canary0 = Referenceable() introducer.remote_publish_v2(ann_t0, canary0) a = introducer.get_announcements() self.failUnlessEqual(len(a), 1) self.failUnlessIdentical(a[0].canary, canary0) self.failUnlessEqual(a[0].index, ("storage", pks)) self.failUnlessEqual(a[0].announcement["app-versions"], app_versions) self.failUnlessEqual(a[0].nickname, u"nick-v2") self.failUnlessEqual(a[0].service_name, "storage") self.failUnlessEqual(a[0].version, "my_version") self.failUnlessEqual(a[0].announcement["anonymous-storage-FURL"], furl1)
def _restart_introducer(_ign): log.msg("restarting introducer") self.create_tub(self.central_portnum) # reset counters for i in range(NUM_CLIENTS): c = subscribing_clients[i] for k in c._debug_counts: c._debug_counts[k] = 0 expected_announcements[i] += 1 # new 'storage' for everyone introducer = IntroducerService() self.the_introducer = introducer newfurl = self.central_tub.registerReference(self.the_introducer, furlFile=iff) assert newfurl == self.introducer_furl
def test_duplicate(self): i = IntroducerService() ic1 = IntroducerClient(None, "introducer.furl", u"my_nickname", "ver23", "oldest_version", {}, realseq, FilePath(self.mktemp())) furl1 = "pb://[email protected]:36106/gydnp" privkey_s, _ = keyutil.make_keypair() privkey, _ = keyutil.parse_privkey(privkey_s) ann1 = make_ann_t(ic1, furl1, privkey, seqnum=10) ann1_old = make_ann_t(ic1, furl1, privkey, seqnum=9) ann1_new = make_ann_t(ic1, furl1, privkey, seqnum=11) ann1_noseqnum = make_ann_t(ic1, furl1, privkey, seqnum=None) ann1_badseqnum = make_ann_t(ic1, furl1, privkey, seqnum="not an int") i.remote_publish_v2(ann1, None) all = i.get_announcements() self.failUnlessEqual(len(all), 1) self.failUnlessEqual(all[0].announcement["seqnum"], 10) self.failUnlessEqual(i._debug_counts["inbound_message"], 1) self.failUnlessEqual(i._debug_counts["inbound_duplicate"], 0) self.failUnlessEqual(i._debug_counts["inbound_no_seqnum"], 0) self.failUnlessEqual(i._debug_counts["inbound_old_replay"], 0) self.failUnlessEqual(i._debug_counts["inbound_update"], 0) i.remote_publish_v2(ann1, None) all = i.get_announcements() self.failUnlessEqual(len(all), 1) self.failUnlessEqual(all[0].announcement["seqnum"], 10) self.failUnlessEqual(i._debug_counts["inbound_message"], 2) self.failUnlessEqual(i._debug_counts["inbound_duplicate"], 1) self.failUnlessEqual(i._debug_counts["inbound_no_seqnum"], 0) self.failUnlessEqual(i._debug_counts["inbound_old_replay"], 0) self.failUnlessEqual(i._debug_counts["inbound_update"], 0) i.remote_publish_v2(ann1_old, None) all = i.get_announcements() self.failUnlessEqual(len(all), 1) self.failUnlessEqual(all[0].announcement["seqnum"], 10) self.failUnlessEqual(i._debug_counts["inbound_message"], 3) self.failUnlessEqual(i._debug_counts["inbound_duplicate"], 1) self.failUnlessEqual(i._debug_counts["inbound_no_seqnum"], 0) self.failUnlessEqual(i._debug_counts["inbound_old_replay"], 1) self.failUnlessEqual(i._debug_counts["inbound_update"], 0) i.remote_publish_v2(ann1_new, None) all = i.get_announcements() self.failUnlessEqual(len(all), 1) self.failUnlessEqual(all[0].announcement["seqnum"], 11) self.failUnlessEqual(i._debug_counts["inbound_message"], 4) self.failUnlessEqual(i._debug_counts["inbound_duplicate"], 1) self.failUnlessEqual(i._debug_counts["inbound_no_seqnum"], 0) self.failUnlessEqual(i._debug_counts["inbound_old_replay"], 1) self.failUnlessEqual(i._debug_counts["inbound_update"], 1) i.remote_publish_v2(ann1_noseqnum, None) all = i.get_announcements() self.failUnlessEqual(len(all), 1) self.failUnlessEqual(all[0].announcement["seqnum"], 11) self.failUnlessEqual(i._debug_counts["inbound_message"], 5) self.failUnlessEqual(i._debug_counts["inbound_duplicate"], 1) self.failUnlessEqual(i._debug_counts["inbound_no_seqnum"], 1) self.failUnlessEqual(i._debug_counts["inbound_old_replay"], 1) self.failUnlessEqual(i._debug_counts["inbound_update"], 1) i.remote_publish_v2(ann1_badseqnum, None) all = i.get_announcements() self.failUnlessEqual(len(all), 1) self.failUnlessEqual(all[0].announcement["seqnum"], 11) self.failUnlessEqual(i._debug_counts["inbound_message"], 6) self.failUnlessEqual(i._debug_counts["inbound_duplicate"], 1) self.failUnlessEqual(i._debug_counts["inbound_no_seqnum"], 2) self.failUnlessEqual(i._debug_counts["inbound_old_replay"], 1) self.failUnlessEqual(i._debug_counts["inbound_update"], 1)
def do_system_test(self): self.create_tub() introducer = IntroducerService() introducer.setServiceParent(self.parent) iff = os.path.join(self.basedir, "introducer.furl") tub = self.central_tub ifurl = self.central_tub.registerReference(introducer, furlFile=iff) self.introducer_furl = ifurl # we have 5 clients who publish themselves as storage servers, and a # sixth which does which not. All 6 clients subscriber to hear about # storage. When the connections are fully established, all six nodes # should have 5 connections each. NUM_STORAGE = 5 NUM_CLIENTS = 6 clients = [] tubs = {} received_announcements = {} subscribing_clients = [] publishing_clients = [] printable_serverids = {} self.the_introducer = introducer privkeys = {} pubkeys = {} expected_announcements = [0 for c in range(NUM_CLIENTS)] for i in range(NUM_CLIENTS): tub = Tub() #tub.setOption("logLocalFailures", True) #tub.setOption("logRemoteFailures", True) tub.setOption("expose-remote-exception-types", False) tub.setServiceParent(self.parent) listenOnUnused(tub) log.msg("creating client %d: %s" % (i, tub.getShortTubID())) c = IntroducerClient(tub, self.introducer_furl, NICKNAME % str(i), "version", "oldest", {"component": "component-v1"}, fakeseq, FilePath(self.mktemp())) received_announcements[c] = {} def got(key_s_or_tubid, ann, announcements): index = key_s_or_tubid or get_tubid_string_from_ann(ann) announcements[index] = ann c.subscribe_to("storage", got, received_announcements[c]) subscribing_clients.append(c) expected_announcements[ i] += 1 # all expect a 'storage' announcement node_furl = tub.registerReference(Referenceable()) privkey_s, pubkey_s = keyutil.make_keypair() privkey, _ignored = keyutil.parse_privkey(privkey_s) privkeys[i] = privkey pubkeys[i] = pubkey_s if i < NUM_STORAGE: # sign all announcements c.publish("storage", make_ann(node_furl), privkey) assert pubkey_s.startswith("pub-") printable_serverids[i] = pubkey_s[len("pub-"):] publishing_clients.append(c) else: # the last one does not publish anything pass if i == 2: # also publish something that nobody cares about boring_furl = tub.registerReference(Referenceable()) c.publish("boring", make_ann(boring_furl), privkey) c.setServiceParent(self.parent) clients.append(c) tubs[c] = tub def _wait_for_connected(ign): def _connected(): for c in clients: if not c.connected_to_introducer(): return False return True return self.poll(_connected) # we watch the clients to determine when the system has settled down. # Then we can look inside the server to assert things about its # state. def _wait_for_expected_announcements(ign): def _got_expected_announcements(): for i, c in enumerate(subscribing_clients): if len(received_announcements[c] ) < expected_announcements[i]: return False return True return self.poll(_got_expected_announcements) # before shutting down any Tub, we'd like to know that there are no # messages outstanding def _wait_until_idle(ign): def _idle(): for c in subscribing_clients + publishing_clients: if c._debug_outstanding: return False if self.the_introducer._debug_outstanding: return False return True return self.poll(_idle) d = defer.succeed(None) d.addCallback(_wait_for_connected) d.addCallback(_wait_for_expected_announcements) d.addCallback(_wait_until_idle) def _check1(res): log.msg("doing _check1") dc = self.the_introducer._debug_counts # each storage server publishes a record. There is also one # "boring" self.failUnlessEqual(dc["inbound_message"], NUM_STORAGE + 1) self.failUnlessEqual(dc["inbound_duplicate"], 0) self.failUnlessEqual(dc["inbound_update"], 0) self.failUnlessEqual(dc["inbound_subscribe"], NUM_CLIENTS) # the number of outbound messages is tricky.. I think it depends # upon a race between the publish and the subscribe messages. self.failUnless(dc["outbound_message"] > 0) # each client subscribes to "storage", and each server publishes self.failUnlessEqual(dc["outbound_announcements"], NUM_STORAGE * NUM_CLIENTS) for c in subscribing_clients: cdc = c._debug_counts self.failUnless(cdc["inbound_message"]) self.failUnlessEqual(cdc["inbound_announcement"], NUM_STORAGE) self.failUnlessEqual(cdc["wrong_service"], 0) self.failUnlessEqual(cdc["duplicate_announcement"], 0) self.failUnlessEqual(cdc["update"], 0) self.failUnlessEqual(cdc["new_announcement"], NUM_STORAGE) anns = received_announcements[c] self.failUnlessEqual(len(anns), NUM_STORAGE) serverid0 = printable_serverids[0] ann = anns[serverid0] nick = ann["nickname"] self.failUnlessEqual(type(nick), unicode) self.failUnlessEqual(nick, NICKNAME % "0") for c in publishing_clients: cdc = c._debug_counts expected = 1 if c in [ clients[2], # boring ]: expected = 2 self.failUnlessEqual(cdc["outbound_message"], expected) # now check the web status, make sure it renders without error ir = introweb.IntroducerRoot(self.parent) self.parent.nodeid = "NODEID" text = ir.renderSynchronously().decode("utf-8") self.failUnlessIn(NICKNAME % "0", text) # a v2 client self.failUnlessIn(NICKNAME % "1", text) # another v2 client for i in range(NUM_STORAGE): self.failUnlessIn(printable_serverids[i], text, (i, printable_serverids[i], text)) # make sure there isn't a double-base32ed string too self.failIfIn(idlib.nodeid_b2a(printable_serverids[i]), text, (i, printable_serverids[i], text)) log.msg("_check1 done") d.addCallback(_check1) # force an introducer reconnect, by shutting down the Tub it's using # and starting a new Tub (with the old introducer). Everybody should # reconnect and republish, but the introducer should ignore the # republishes as duplicates. However, because the server doesn't know # what each client does and does not know, it will send them a copy # of the current announcement table anyway. d.addCallback(lambda _ign: log.msg("shutting down introducer's Tub")) d.addCallback(lambda _ign: self.central_tub.disownServiceParent()) def _wait_for_introducer_loss(ign): def _introducer_lost(): for c in clients: if c.connected_to_introducer(): return False return True return self.poll(_introducer_lost) d.addCallback(_wait_for_introducer_loss) def _restart_introducer_tub(_ign): log.msg("restarting introducer's Tub") # reset counters for i in range(NUM_CLIENTS): c = subscribing_clients[i] for k in c._debug_counts: c._debug_counts[k] = 0 for k in self.the_introducer._debug_counts: self.the_introducer._debug_counts[k] = 0 expected_announcements[i] += 1 # new 'storage' for everyone self.create_tub(self.central_portnum) newfurl = self.central_tub.registerReference(self.the_introducer, furlFile=iff) assert newfurl == self.introducer_furl d.addCallback(_restart_introducer_tub) d.addCallback(_wait_for_connected) d.addCallback(_wait_for_expected_announcements) d.addCallback(_wait_until_idle) d.addCallback(lambda _ign: log.msg(" reconnected")) # TODO: publish something while the introducer is offline, then # confirm it gets delivered when the connection is reestablished def _check2(res): log.msg("doing _check2") # assert that the introducer sent out new messages, one per # subscriber dc = self.the_introducer._debug_counts self.failUnlessEqual(dc["outbound_announcements"], NUM_STORAGE * NUM_CLIENTS) self.failUnless(dc["outbound_message"] > 0) self.failUnlessEqual(dc["inbound_subscribe"], NUM_CLIENTS) for c in subscribing_clients: cdc = c._debug_counts self.failUnlessEqual(cdc["inbound_message"], 1) self.failUnlessEqual(cdc["inbound_announcement"], NUM_STORAGE) self.failUnlessEqual(cdc["new_announcement"], 0) self.failUnlessEqual(cdc["wrong_service"], 0) self.failUnlessEqual(cdc["duplicate_announcement"], NUM_STORAGE) d.addCallback(_check2) # Then force an introducer restart, by shutting down the Tub, # destroying the old introducer, and starting a new Tub+Introducer. # Everybody should reconnect and republish, and the (new) introducer # will distribute the new announcements, but the clients should # ignore the republishes as duplicates. d.addCallback(lambda _ign: log.msg("shutting down introducer")) d.addCallback(lambda _ign: self.central_tub.disownServiceParent()) d.addCallback(_wait_for_introducer_loss) d.addCallback(lambda _ign: log.msg("introducer lost")) def _restart_introducer(_ign): log.msg("restarting introducer") self.create_tub(self.central_portnum) # reset counters for i in range(NUM_CLIENTS): c = subscribing_clients[i] for k in c._debug_counts: c._debug_counts[k] = 0 expected_announcements[i] += 1 # new 'storage' for everyone introducer = IntroducerService() self.the_introducer = introducer newfurl = self.central_tub.registerReference(self.the_introducer, furlFile=iff) assert newfurl == self.introducer_furl d.addCallback(_restart_introducer) d.addCallback(_wait_for_connected) d.addCallback(_wait_for_expected_announcements) d.addCallback(_wait_until_idle) def _check3(res): log.msg("doing _check3") dc = self.the_introducer._debug_counts self.failUnlessEqual(dc["outbound_announcements"], NUM_STORAGE * NUM_CLIENTS) self.failUnless(dc["outbound_message"] > 0) self.failUnlessEqual(dc["inbound_subscribe"], NUM_CLIENTS) for c in subscribing_clients: cdc = c._debug_counts self.failUnless(cdc["inbound_message"] > 0) self.failUnlessEqual(cdc["inbound_announcement"], NUM_STORAGE) self.failUnlessEqual(cdc["new_announcement"], 0) self.failUnlessEqual(cdc["wrong_service"], 0) self.failUnlessEqual(cdc["duplicate_announcement"], NUM_STORAGE) d.addCallback(_check3) return d
def test_duplicate(self): i = IntroducerService() self.failUnlessEqual(len(i.get_announcements()), 0) self.failUnlessEqual(len(i.get_subscribers()), 0) furl1 = "pb://[email protected]:36106,127.0.0.1:36106/gydnpigj2ja2qr2srq4ikjwnl7xfgbra" furl2 = "pb://[email protected]:36111,127.0.0.1:36106/ttwwoogj2ja2qr2srq4ikjwnl7xfgbra" ann1 = (furl1, "storage", "RIStorage", "nick1", "ver23", "ver0") ann1b = (furl1, "storage", "RIStorage", "nick1", "ver24", "ver0") ann2 = (furl2, "storage", "RIStorage", "nick2", "ver30", "ver0") i.remote_publish(ann1) self.failUnlessEqual(len(i.get_announcements()), 1) self.failUnlessEqual(len(i.get_subscribers()), 0) i.remote_publish(ann2) self.failUnlessEqual(len(i.get_announcements()), 2) self.failUnlessEqual(len(i.get_subscribers()), 0) i.remote_publish(ann1b) self.failUnlessEqual(len(i.get_announcements()), 2) self.failUnlessEqual(len(i.get_subscribers()), 0)
def test_listen(self): i = IntroducerService() i.setServiceParent(self.parent)
def do_system_test(self): self.create_tub() introducer = IntroducerService() introducer.setServiceParent(self.parent) iff = os.path.join(self.basedir, "introducer.furl") tub = self.central_tub ifurl = self.central_tub.registerReference(introducer, furlFile=iff) self.introducer_furl = ifurl # we have 5 clients who publish themselves as storage servers, and a # sixth which does which not. All 6 clients subscriber to hear about # storage. When the connections are fully established, all six nodes # should have 5 connections each. NUM_STORAGE = 5 NUM_CLIENTS = 6 clients = [] tubs = {} received_announcements = {} subscribing_clients = [] publishing_clients = [] printable_serverids = {} self.the_introducer = introducer privkeys = {} pubkeys = {} expected_announcements = [0 for c in range(NUM_CLIENTS)] for i in range(NUM_CLIENTS): tub = Tub() #tub.setOption("logLocalFailures", True) #tub.setOption("logRemoteFailures", True) tub.setOption("expose-remote-exception-types", False) tub.setServiceParent(self.parent) portnum = iputil.allocate_tcp_port() tub.listenOn("tcp:%d" % portnum) tub.setLocation("localhost:%d" % portnum) log.msg("creating client %d: %s" % (i, tub.getShortTubID())) c = IntroducerClient(tub, self.introducer_furl, NICKNAME % str(i), "version", "oldest", {"component": "component-v1"}, fakeseq, FilePath(self.mktemp())) received_announcements[c] = {} def got(key_s_or_tubid, ann, announcements): index = key_s_or_tubid or get_tubid_string_from_ann(ann) announcements[index] = ann c.subscribe_to("storage", got, received_announcements[c]) subscribing_clients.append(c) expected_announcements[i] += 1 # all expect a 'storage' announcement node_furl = tub.registerReference(Referenceable()) privkey_s, pubkey_s = keyutil.make_keypair() privkey, _ignored = keyutil.parse_privkey(privkey_s) privkeys[i] = privkey pubkeys[i] = pubkey_s if i < NUM_STORAGE: # sign all announcements c.publish("storage", make_ann(node_furl), privkey) assert pubkey_s.startswith("pub-") printable_serverids[i] = pubkey_s[len("pub-"):] publishing_clients.append(c) else: # the last one does not publish anything pass if i == 2: # also publish something that nobody cares about boring_furl = tub.registerReference(Referenceable()) c.publish("boring", make_ann(boring_furl), privkey) c.setServiceParent(self.parent) clients.append(c) tubs[c] = tub def _wait_for_connected(ign): def _connected(): for c in clients: if not c.connected_to_introducer(): return False return True return self.poll(_connected) # we watch the clients to determine when the system has settled down. # Then we can look inside the server to assert things about its # state. def _wait_for_expected_announcements(ign): def _got_expected_announcements(): for i,c in enumerate(subscribing_clients): if len(received_announcements[c]) < expected_announcements[i]: return False return True return self.poll(_got_expected_announcements) # before shutting down any Tub, we'd like to know that there are no # messages outstanding def _wait_until_idle(ign): def _idle(): for c in subscribing_clients + publishing_clients: if c._debug_outstanding: return False if self.the_introducer._debug_outstanding: return False return True return self.poll(_idle) d = defer.succeed(None) d.addCallback(_wait_for_connected) d.addCallback(_wait_for_expected_announcements) d.addCallback(_wait_until_idle) def _check1(res): log.msg("doing _check1") dc = self.the_introducer._debug_counts # each storage server publishes a record. There is also one # "boring" self.failUnlessEqual(dc["inbound_message"], NUM_STORAGE+1) self.failUnlessEqual(dc["inbound_duplicate"], 0) self.failUnlessEqual(dc["inbound_update"], 0) self.failUnlessEqual(dc["inbound_subscribe"], NUM_CLIENTS) # the number of outbound messages is tricky.. I think it depends # upon a race between the publish and the subscribe messages. self.failUnless(dc["outbound_message"] > 0) # each client subscribes to "storage", and each server publishes self.failUnlessEqual(dc["outbound_announcements"], NUM_STORAGE*NUM_CLIENTS) for c in subscribing_clients: cdc = c._debug_counts self.failUnless(cdc["inbound_message"]) self.failUnlessEqual(cdc["inbound_announcement"], NUM_STORAGE) self.failUnlessEqual(cdc["wrong_service"], 0) self.failUnlessEqual(cdc["duplicate_announcement"], 0) self.failUnlessEqual(cdc["update"], 0) self.failUnlessEqual(cdc["new_announcement"], NUM_STORAGE) anns = received_announcements[c] self.failUnlessEqual(len(anns), NUM_STORAGE) serverid0 = printable_serverids[0] ann = anns[serverid0] nick = ann["nickname"] self.failUnlessEqual(type(nick), unicode) self.failUnlessEqual(nick, NICKNAME % "0") for c in publishing_clients: cdc = c._debug_counts expected = 1 if c in [clients[2], # boring ]: expected = 2 self.failUnlessEqual(cdc["outbound_message"], expected) # now check the web status, make sure it renders without error ir = introweb.IntroducerRoot(self.parent) self.parent.nodeid = "NODEID" text = ir.renderSynchronously().decode("utf-8") self.failUnlessIn(NICKNAME % "0", text) # a v2 client self.failUnlessIn(NICKNAME % "1", text) # another v2 client for i in range(NUM_STORAGE): self.failUnlessIn(printable_serverids[i], text, (i,printable_serverids[i],text)) # make sure there isn't a double-base32ed string too self.failIfIn(idlib.nodeid_b2a(printable_serverids[i]), text, (i,printable_serverids[i],text)) log.msg("_check1 done") d.addCallback(_check1) # force an introducer reconnect, by shutting down the Tub it's using # and starting a new Tub (with the old introducer). Everybody should # reconnect and republish, but the introducer should ignore the # republishes as duplicates. However, because the server doesn't know # what each client does and does not know, it will send them a copy # of the current announcement table anyway. d.addCallback(lambda _ign: log.msg("shutting down introducer's Tub")) d.addCallback(lambda _ign: self.central_tub.disownServiceParent()) def _wait_for_introducer_loss(ign): def _introducer_lost(): for c in clients: if c.connected_to_introducer(): return False return True return self.poll(_introducer_lost) d.addCallback(_wait_for_introducer_loss) def _restart_introducer_tub(_ign): log.msg("restarting introducer's Tub") # reset counters for i in range(NUM_CLIENTS): c = subscribing_clients[i] for k in c._debug_counts: c._debug_counts[k] = 0 for k in self.the_introducer._debug_counts: self.the_introducer._debug_counts[k] = 0 expected_announcements[i] += 1 # new 'storage' for everyone self.create_tub(self.central_portnum) newfurl = self.central_tub.registerReference(self.the_introducer, furlFile=iff) assert newfurl == self.introducer_furl d.addCallback(_restart_introducer_tub) d.addCallback(_wait_for_connected) d.addCallback(_wait_for_expected_announcements) d.addCallback(_wait_until_idle) d.addCallback(lambda _ign: log.msg(" reconnected")) # TODO: publish something while the introducer is offline, then # confirm it gets delivered when the connection is reestablished def _check2(res): log.msg("doing _check2") # assert that the introducer sent out new messages, one per # subscriber dc = self.the_introducer._debug_counts self.failUnlessEqual(dc["outbound_announcements"], NUM_STORAGE*NUM_CLIENTS) self.failUnless(dc["outbound_message"] > 0) self.failUnlessEqual(dc["inbound_subscribe"], NUM_CLIENTS) for c in subscribing_clients: cdc = c._debug_counts self.failUnlessEqual(cdc["inbound_message"], 1) self.failUnlessEqual(cdc["inbound_announcement"], NUM_STORAGE) self.failUnlessEqual(cdc["new_announcement"], 0) self.failUnlessEqual(cdc["wrong_service"], 0) self.failUnlessEqual(cdc["duplicate_announcement"], NUM_STORAGE) d.addCallback(_check2) # Then force an introducer restart, by shutting down the Tub, # destroying the old introducer, and starting a new Tub+Introducer. # Everybody should reconnect and republish, and the (new) introducer # will distribute the new announcements, but the clients should # ignore the republishes as duplicates. d.addCallback(lambda _ign: log.msg("shutting down introducer")) d.addCallback(lambda _ign: self.central_tub.disownServiceParent()) d.addCallback(_wait_for_introducer_loss) d.addCallback(lambda _ign: log.msg("introducer lost")) def _restart_introducer(_ign): log.msg("restarting introducer") self.create_tub(self.central_portnum) # reset counters for i in range(NUM_CLIENTS): c = subscribing_clients[i] for k in c._debug_counts: c._debug_counts[k] = 0 expected_announcements[i] += 1 # new 'storage' for everyone introducer = IntroducerService() self.the_introducer = introducer newfurl = self.central_tub.registerReference(self.the_introducer, furlFile=iff) assert newfurl == self.introducer_furl d.addCallback(_restart_introducer) d.addCallback(_wait_for_connected) d.addCallback(_wait_for_expected_announcements) d.addCallback(_wait_until_idle) def _check3(res): log.msg("doing _check3") dc = self.the_introducer._debug_counts self.failUnlessEqual(dc["outbound_announcements"], NUM_STORAGE*NUM_CLIENTS) self.failUnless(dc["outbound_message"] > 0) self.failUnlessEqual(dc["inbound_subscribe"], NUM_CLIENTS) for c in subscribing_clients: cdc = c._debug_counts self.failUnless(cdc["inbound_message"] > 0) self.failUnlessEqual(cdc["inbound_announcement"], NUM_STORAGE) self.failUnlessEqual(cdc["new_announcement"], 0) self.failUnlessEqual(cdc["wrong_service"], 0) self.failUnlessEqual(cdc["duplicate_announcement"], NUM_STORAGE) d.addCallback(_check3) return d