def create_fake_client(self):
sb = StorageFarmBroker(True)
# s.get_name() (the "short description") will be "v0-00000000".
# s.get_longname() will include the -long suffix.
# s.get_peerid() (i.e. tubid) will be "aaa.." or "777.." or "ceir.."
servers = [("v0-00000000-long", "\x00" * 20, "peer-0"),
("v0-ffffffff-long", "\xff" * 20, "peer-f"),
("v0-11111111-long", "\x11" * 20, "peer-11")]
for (key_s, peerid, nickname) in servers:
tubid_b32 = base32.b2a(peerid)
furl = "pb://%s@nowhere/fake" % tubid_b32
ann = {
"version": 0,
"service-name": "storage",
"anonymous-storage-FURL": furl,
"permutation-seed-base32": "",
"nickname": unicode(nickname),
"app-versions": {}, # need #466 and v2 introducer
"my-version": "ver",
"oldest-supported": "oldest",
}
s = NativeStorageServer(key_s, ann)
sb.test_add_server(peerid, s) # XXX: maybe use key_s?
c = FakeClient()
c.storage_broker = sb
return c
def create_fake_client(self):
sb = StorageFarmBroker(True, None, EMPTY_CLIENT_CONFIG)
# s.get_name() (the "short description") will be "v0-00000000".
# s.get_longname() will include the -long suffix.
servers = [("v0-00000000-long", "\x00" * 20, "peer-0"),
("v0-ffffffff-long", "\xff" * 20, "peer-f"),
("v0-11111111-long", "\x11" * 20, "peer-11")]
for (key_s, binary_tubid, nickname) in servers:
server_id = key_s
tubid_b32 = base32.b2a(binary_tubid)
furl = "pb://%s@nowhere/fake" % tubid_b32
ann = {
"version": 0,
"service-name": "storage",
"anonymous-storage-FURL": furl,
"permutation-seed-base32": "",
"nickname": unicode(nickname),
"app-versions": {}, # need #466 and v2 introducer
"my-version": "ver",
"oldest-supported": "oldest",
}
s = NativeStorageServer(server_id, ann, None, None, None)
sb.test_add_server(server_id, s)
c = FakeClient()
c.storage_broker = sb
return c
def test_static_servers(self):
broker = StorageFarmBroker(True, lambda h: Mock())
key_s = 'v0-1234-1'
servers_yaml = """\
storage:
v0-1234-1:
ann:
anonymous-storage-FURL: pb://ge@nowhere/fake
permutation-seed-base32: aaaaaaaaaaaaaaaaaaaaaaaa
"""
servers = yamlutil.safe_load(servers_yaml)
permseed = base32.a2b("aaaaaaaaaaaaaaaaaaaaaaaa")
broker.set_static_servers(servers["storage"])
self.failUnlessEqual(len(broker._static_server_ids), 1)
s = broker.servers[key_s]
self.failUnlessEqual(s.announcement,
servers["storage"]["v0-1234-1"]["ann"])
self.failUnlessEqual(s.get_serverid(), key_s)
self.assertEqual(s.get_permutation_seed(), permseed)
# if the Introducer announces the same thing, we're supposed to
# ignore it
ann2 = {
"service-name": "storage",
"anonymous-storage-FURL": "pb://{}@nowhere/fake2".format(base32.b2a(str(1))),
"permutation-seed-base32": "bbbbbbbbbbbbbbbbbbbbbbbb",
}
broker._got_announcement(key_s, ann2)
s2 = broker.servers[key_s]
self.assertIdentical(s2, s)
self.assertEqual(s2.get_permutation_seed(), permseed)
def create_fake_client(self):
sb = StorageFarmBroker(None, True)
# s.get_name() (the "short description") will be "v0-00000000".
# s.get_longname() will include the -long suffix.
# s.get_peerid() (i.e. tubid) will be "aaa.." or "777.." or "ceir.."
servers = [("v0-00000000-long", "\x00"*20, "peer-0"),
("v0-ffffffff-long", "\xff"*20, "peer-f"),
("v0-11111111-long", "\x11"*20, "peer-11")]
for (key_s, peerid, nickname) in servers:
tubid_b32 = base32.b2a(peerid)
furl = "pb://%s@nowhere/fake" % tubid_b32
ann = { "version": 0,
"service-name": "storage",
"anonymous-storage-FURL": furl,
"permutation-seed-base32": "",
"nickname": unicode(nickname),
"app-versions": {}, # need #466 and v2 introducer
"my-version": "ver",
"oldest-supported": "oldest",
}
s = NativeStorageServer(key_s, ann)
sb.test_add_server(peerid, s) # XXX: maybe use key_s?
c = FakeClient()
c.storage_broker = sb
return c
def test_permute(self):
sb = StorageFarmBroker(None, True)
for k in ["%d" % i for i in range(5)]:
sb.test_add_rref(k, "rref")
self.failUnlessReallyEqual(self._permute(sb, "one"), ['3','1','0','4','2'])
self.failUnlessReallyEqual(self._permute(sb, "two"), ['0','4','2','1','3'])
sb.servers.clear()
self.failUnlessReallyEqual(self._permute(sb, "one"), [])
def test_static_permutation_seed_hashed(self):
broker = StorageFarmBroker(True, lambda h: Mock())
server_id = "unparseable"
ann = {
"anonymous-storage-FURL": "pb://abcde@nowhere/fake",
}
broker.set_static_servers({server_id.decode("ascii"): {"ann": ann}})
s = broker.servers[server_id]
self.assertEqual(s.get_permutation_seed(),
hashlib.sha256(server_id).digest())
def test_permute(self):
sb = StorageFarmBroker(None, True)
for k in ["%d" % i for i in range(5)]:
ann = {"anonymous-storage-FURL": "pb://abcde@nowhere/fake", "permutation-seed-base32": base32.b2a(k)}
sb.test_add_rref(k, "rref", ann)
self.failUnlessReallyEqual(self._permute(sb, "one"), ["3", "1", "0", "4", "2"])
self.failUnlessReallyEqual(self._permute(sb, "two"), ["0", "4", "2", "1", "3"])
sb.servers.clear()
self.failUnlessReallyEqual(self._permute(sb, "one"), [])
def test_static_permutation_seed_pubkey(self):
broker = StorageFarmBroker(True, lambda h: Mock())
server_id = "v0-4uazse3xb6uu5qpkb7tel2bm6bpea4jhuigdhqcuvvse7hugtsia"
k = "4uazse3xb6uu5qpkb7tel2bm6bpea4jhuigdhqcuvvse7hugtsia"
ann = {
"anonymous-storage-FURL": "pb://abcde@nowhere/fake",
}
broker.set_static_servers({server_id.decode("ascii"): {"ann": ann}})
s = broker.servers[server_id]
self.assertEqual(s.get_permutation_seed(), base32.a2b(k))
def test_static_permutation_seed_pubkey(self):
broker = StorageFarmBroker(True, lambda h: Mock())
server_id = "v0-4uazse3xb6uu5qpkb7tel2bm6bpea4jhuigdhqcuvvse7hugtsia"
k = "4uazse3xb6uu5qpkb7tel2bm6bpea4jhuigdhqcuvvse7hugtsia"
ann = {
"anonymous-storage-FURL": "pb://abcde@nowhere/fake",
}
broker.set_static_servers({server_id.decode("ascii"): {"ann": ann}})
s = broker.servers[server_id]
self.assertEqual(s.get_permutation_seed(), base32.a2b(k))
def test_static_permutation_seed_hashed(self):
broker = StorageFarmBroker(True, lambda h: Mock())
server_id = "unparseable"
ann = {
"anonymous-storage-FURL": "pb://abcde@nowhere/fake",
}
broker.set_static_servers({server_id.decode("ascii"): {"ann": ann}})
s = broker.servers[server_id]
self.assertEqual(s.get_permutation_seed(),
hashlib.sha256(server_id).digest())
def test_permute_with_preferred(self):
sb = StorageFarmBroker(True, None, preferred_peers=['1','4'])
for k in ["%d" % i for i in range(5)]:
ann = {"anonymous-storage-FURL": "pb://abcde@nowhere/fake",
"permutation-seed-base32": base32.b2a(k) }
sb.test_add_rref(k, "rref", ann)
self.failUnlessReallyEqual(self._permute(sb, "one"), ['1','4','3','0','2'])
self.failUnlessReallyEqual(self._permute(sb, "two"), ['4','1','0','2','3'])
sb.servers.clear()
self.failUnlessReallyEqual(self._permute(sb, "one"), [])
def test_static_permutation_seed_explicit(self):
broker = StorageFarmBroker(True, lambda h: Mock())
server_id = "v0-4uazse3xb6uu5qpkb7tel2bm6bpea4jhuigdhqcuvvse7hugtsia"
k = "w5gl5igiexhwmftwzhai5jy2jixn7yx7"
ann = {
"anonymous-storage-FURL": "pb://abcde@nowhere/fake",
"permutation-seed-base32": k,
}
broker.set_static_servers({server_id.decode("ascii"): {"ann": ann}})
s = broker.servers[server_id]
self.assertEqual(s.get_permutation_seed(), base32.a2b(k))
def test_permute(self):
sb = StorageFarmBroker(None, True)
for k in ["%d" % i for i in range(5)]:
sb.test_add_rref(k, "rref")
self.failUnlessReallyEqual(self._permute(sb, "one"),
['3', '1', '0', '4', '2'])
self.failUnlessReallyEqual(self._permute(sb, "two"),
['0', '4', '2', '1', '3'])
sb.servers.clear()
self.failUnlessReallyEqual(self._permute(sb, "one"), [])
def test_static_permutation_seed_explicit(self):
broker = StorageFarmBroker(True, lambda h: Mock())
server_id = "v0-4uazse3xb6uu5qpkb7tel2bm6bpea4jhuigdhqcuvvse7hugtsia"
k = "w5gl5igiexhwmftwzhai5jy2jixn7yx7"
ann = {
"anonymous-storage-FURL": "pb://abcde@nowhere/fake",
"permutation-seed-base32": k,
}
broker.set_static_servers({server_id.decode("ascii"): {"ann": ann}})
s = broker.servers[server_id]
self.assertEqual(s.get_permutation_seed(), base32.a2b(k))
def test_permute_with_preferred(self):
sb = StorageFarmBroker(True, ['1','4'])
for k in ["%d" % i for i in range(5)]:
ann = {"anonymous-storage-FURL": "pb://abcde@nowhere/fake",
"permutation-seed-base32": base32.b2a(k) }
sb.test_add_rref(k, "rref", ann)
self.failUnlessReallyEqual(self._permute(sb, "one"), ['1','4','3','0','2'])
self.failUnlessReallyEqual(self._permute(sb, "two"), ['4','1','0','2','3'])
sb.servers.clear()
self.failUnlessReallyEqual(self._permute(sb, "one"), [])
def make_storagebroker(s=None, num_peers=10):
if not s:
s = FakeStorage()
peerids = [tagged_hash("peerid", "%d" % i)[:20]
for i in range(num_peers)]
storage_broker = StorageFarmBroker(True, None)
for peerid in peerids:
fss = FakeStorageServer(peerid, s)
ann = {"anonymous-storage-FURL": "pb://%s@nowhere/fake" % base32.b2a(peerid),
"permutation-seed-base32": base32.b2a(peerid) }
storage_broker.test_add_rref(peerid, fss, ann)
return storage_broker
def test_threshold_reached(self):
introducer = Mock()
new_tubs = []
def make_tub(*args, **kwargs):
return new_tubs.pop()
broker = StorageFarmBroker(True, make_tub)
done = broker.when_connected_enough(5)
broker.use_introducer(introducer)
# subscribes to "storage" to learn of new storage nodes
subscribe = introducer.mock_calls[0]
self.assertEqual(subscribe[0], 'subscribe_to')
self.assertEqual(subscribe[1][0], 'storage')
got_announcement = subscribe[1][1]
data = {
"service-name": "storage",
"anonymous-storage-FURL": None,
"permutation-seed-base32": "aaaaaaaaaaaaaaaaaaaaaaaa",
}
def add_one_server(x):
data["anonymous-storage-FURL"] = "pb://{}@nowhere/fake".format(
base32.b2a(str(x)))
tub = Mock()
new_tubs.append(tub)
got_announcement('v0-1234-{}'.format(x), data)
self.assertEqual(tub.mock_calls[-1][0], 'connectTo')
got_connection = tub.mock_calls[-1][1][1]
rref = Mock()
rref.callRemote = Mock(return_value=succeed(1234))
got_connection(rref)
# first 4 shouldn't trigger connected_threashold
for x in range(4):
add_one_server(x)
self.assertFalse(done.called)
# ...but the 5th *should* trigger the threshold
add_one_server(42)
# so: the OneShotObserverList only notifies via
# foolscap.eventually() -- which forces the Deferred call
# through the reactor -- so it's no longer synchronous,
# meaning that we have to do "real reactor stuff" for the
# Deferred from when_connected_enough() to actually fire. (or
# @patch() out the reactor in foolscap.eventually to be a
# Clock() so we can advance time ourselves, but ... luckily
# eventually() uses 0 as the timeout currently)
yield done
self.assertTrue(done.called)
def test_threshold_reached(self):
introducer = Mock()
new_tubs = []
def make_tub(*args, **kwargs):
return new_tubs.pop()
broker = StorageFarmBroker(True, make_tub)
done = broker.when_connected_enough(5)
broker.use_introducer(introducer)
# subscribes to "storage" to learn of new storage nodes
subscribe = introducer.mock_calls[0]
self.assertEqual(subscribe[0], 'subscribe_to')
self.assertEqual(subscribe[1][0], 'storage')
got_announcement = subscribe[1][1]
data = {
"service-name": "storage",
"anonymous-storage-FURL": None,
"permutation-seed-base32": "aaaaaaaaaaaaaaaaaaaaaaaa",
}
def add_one_server(x):
data["anonymous-storage-FURL"] = "pb://{}@nowhere/fake".format(base32.b2a(str(x)))
tub = Mock()
new_tubs.append(tub)
got_announcement('v0-1234-{}'.format(x), data)
self.assertEqual(tub.mock_calls[-1][0], 'connectTo')
got_connection = tub.mock_calls[-1][1][1]
rref = Mock()
rref.callRemote = Mock(return_value=succeed(1234))
got_connection(rref)
# first 4 shouldn't trigger connected_threashold
for x in range(4):
add_one_server(x)
self.assertFalse(done.called)
# ...but the 5th *should* trigger the threshold
add_one_server(42)
# so: the OneShotObserverList only notifies via
# foolscap.eventually() -- which forces the Deferred call
# through the reactor -- so it's no longer synchronous,
# meaning that we have to do "real reactor stuff" for the
# Deferred from when_connected_enough() to actually fire. (or
# @patch() out the reactor in foolscap.eventually to be a
# Clock() so we can advance time ourselves, but ... luckily
# eventually() uses 0 as the timeout currently)
yield done
self.assertTrue(done.called)
def make_storagebroker(s=None, num_peers=10):
if not s:
s = FakeStorage()
peerids = [tagged_hash("peerid", "%d" % i)[:20] for i in range(num_peers)]
storage_broker = StorageFarmBroker(True)
for peerid in peerids:
fss = FakeStorageServer(peerid, s)
ann = {
"anonymous-storage-FURL":
"pb://%s@nowhere/fake" % base32.b2a(peerid),
"permutation-seed-base32": base32.b2a(peerid)
}
storage_broker.test_add_rref(peerid, fss, ann)
return storage_broker
def make_storagebroker_with_peers(peers):
"""
Make a ``StorageFarmBroker`` connected to the given storage servers.
:param list peers: The storage servers to associate with the storage
broker.
"""
storage_broker = StorageFarmBroker(True, None, EMPTY_CLIENT_CONFIG)
for peer in peers:
storage_broker.test_add_rref(
peer.peerid,
peer.storage_server,
peer.announcement,
)
return storage_broker
def test_static_announcement(self):
broker = StorageFarmBroker(True)
key_s = 'v0-1234-{}'.format(1)
ann = {
"service-name":
"storage",
"anonymous-storage-FURL":
"pb://{}@nowhere/fake".format(base32.b2a(str(1))),
"permutation-seed-base32":
"aaaaaaaaaaaaaaaaaaaaaaaa",
}
broker.got_static_announcement(key_s, ann, None)
self.failUnlessEqual(len(broker.static_servers), 1)
self.failUnlessEqual(broker.servers['1'].announcement, ann)
self.failUnlessEqual(broker.servers['1'].key_s, key_s)
def make_broker(tub_maker=None):
"""
Create a ``StorageFarmBroker`` with the given tub maker and an empty
client configuration.
"""
if tub_maker is None:
tub_maker = lambda handler_overrides: new_tub()
return StorageFarmBroker(True, tub_maker, EMPTY_CLIENT_CONFIG)
def create_fake_client(self):
sb = StorageFarmBroker(None, True)
for (peerid, nickname) in [("\x00"*20, "peer-0"),
("\xff"*20, "peer-f"),
("\x11"*20, "peer-11")] :
ann_d = { "version": 0,
"service-name": "storage",
"FURL": "fake furl",
"nickname": unicode(nickname),
"app-versions": {}, # need #466 and v2 introducer
"my-version": "ver",
"oldest-supported": "oldest",
}
s = NativeStorageServer(peerid, ann_d)
sb.test_add_server(peerid, s)
c = FakeClient()
c.storage_broker = sb
return c
def create_fake_client(self):
sb = StorageFarmBroker(None, True)
for (peerid, nickname) in [("\x00"*20, "peer-0"),
("\xff"*20, "peer-f"),
("\x11"*20, "peer-11")] :
ann_d = { "version": 0,
"service-name": "storage",
"FURL": "fake furl",
"nickname": unicode(nickname),
"app-versions": {}, # need #466 and v2 introducer
"my-version": "ver",
"oldest-supported": "oldest",
}
dsc = NativeStorageClientDescriptor(peerid, ann_d)
sb.test_add_descriptor(peerid, dsc)
c = FakeClient()
c.storage_broker = sb
return c
def create_fake_client(self):
sb = StorageFarmBroker(None, True)
for (peerid, nickname) in [("\x00"*20, "peer-0"),
("\xff"*20, "peer-f"),
("\x11"*20, "peer-11")] :
ann = { "version": 0,
"service-name": "storage",
"anonymous-storage-FURL": "pb://abcde@nowhere/fake",
"permutation-seed-base32": "",
"nickname": unicode(nickname),
"app-versions": {}, # need #466 and v2 introducer
"my-version": "ver",
"oldest-supported": "oldest",
}
s = NativeStorageServer(peerid, ann)
sb.test_add_server(peerid, s)
c = FakeClient()
c.storage_broker = sb
return c
def setUp(self):
self.s = FakeClient()
self.s.storage_broker = StorageFarmBroker(None, True)
self.s.secret_holder = client.SecretHolder("lease secret", "converge")
self.s.startService()
self.tub = t = Tub()
t.setOption("expose-remote-exception-types", False)
t.setServiceParent(self.s)
self.s.tub = t
# we never actually use this for network traffic, so it can use a
# bogus host/port
t.setLocation("bogus:1234")
def test_static_servers(self):
broker = StorageFarmBroker(True, lambda h: Mock())
key_s = 'v0-1234-1'
servers_yaml = """\
storage:
v0-1234-1:
ann:
anonymous-storage-FURL: pb://ge@nowhere/fake
permutation-seed-base32: aaaaaaaaaaaaaaaaaaaaaaaa
"""
servers = yamlutil.safe_load(servers_yaml)
permseed = base32.a2b("aaaaaaaaaaaaaaaaaaaaaaaa")
broker.set_static_servers(servers["storage"])
self.failUnlessEqual(len(broker._static_server_ids), 1)
s = broker.servers[key_s]
self.failUnlessEqual(s.announcement,
servers["storage"]["v0-1234-1"]["ann"])
self.failUnlessEqual(s.get_serverid(), key_s)
self.assertEqual(s.get_permutation_seed(), permseed)
# if the Introducer announces the same thing, we're supposed to
# ignore it
ann2 = {
"service-name":
"storage",
"anonymous-storage-FURL":
"pb://{}@nowhere/fake2".format(base32.b2a(str(1))),
"permutation-seed-base32":
"bbbbbbbbbbbbbbbbbbbbbbbb",
}
broker._got_announcement(key_s, ann2)
s2 = broker.servers[key_s]
self.assertIdentical(s2, s)
self.assertEqual(s2.get_permutation_seed(), permseed)
def make_broker(tub_maker=lambda h: Mock()):
"""
Create a ``StorageFarmBroker`` with the given tub maker and an empty
client configuration.
"""
return StorageFarmBroker(True, tub_maker, EMPTY_CLIENT_CONFIG)
