def setUp(self):
# Do this on a per-test basis so we have a clean ghidcache for the
# restoration test
clear_ghidcache('/ghidcache_test')
self.percore = PersistenceCore()
self.doorman = Doorman()
self.enforcer = Enforcer()
self.lawyer = Lawyer()
self.bookie = Bookie()
self.librarian = DiskLibrarian(cache_dir='/ghidcache_test')
self.postman = MrPostman()
self.undertaker = Undertaker()
self.salmonator = Salmonator()
self.rolodex = RolodexMock()
self.golcore = GCoreMock(TEST_AGENT1)
self.percore.assemble(self.doorman, self.enforcer, self.lawyer,
self.bookie, self.librarian, self.postman,
self.undertaker, self.salmonator)
self.doorman.assemble(self.librarian)
self.postman.assemble(self.golcore, self.librarian, self.bookie,
self.rolodex)
self.undertaker.assemble(self.librarian, self.bookie, self.postman)
self.lawyer.assemble(self.librarian)
self.enforcer.assemble(self.librarian)
self.bookie.assemble(self.librarian, self.lawyer, self.undertaker)
self.librarian.assemble(self.percore)
self.salmonator.assemble(self.percore, self.percore, self.doorman,
self.postman, self.librarian)
def setUp(self):
self.percore = PersistenceCore()
self.doorman = Doorman()
self.enforcer = Enforcer()
self.lawyer = Lawyer()
self.bookie = Bookie()
self.librarian = MemoryLibrarian()
self.postman = MrPostman()
self.undertaker = Undertaker()
self.salmonator = Salmonator()
self.rolodex = RolodexMock()
self.golcore = GCoreMock(TEST_AGENT1)
self.percore.assemble(self.doorman, self.enforcer, self.lawyer,
self.bookie, self.librarian, self.postman,
self.undertaker, self.salmonator)
self.doorman.assemble(self.librarian)
self.postman.assemble(self.golcore, self.librarian, self.bookie,
self.rolodex)
self.undertaker.assemble(self.librarian, self.bookie, self.postman)
self.lawyer.assemble(self.librarian)
self.enforcer.assemble(self.librarian)
self.bookie.assemble(self.librarian, self.lawyer, self.undertaker)
self.librarian.assemble(self.percore)
self.salmonator.assemble(self.percore, self.percore, self.doorman,
self.postman, self.librarian)
def setUp(self):
# These are directly required by the oracle
self.golcore = GolixCore()
self.ghidproxy = GhidProxier()
self.privateer = Privateer()
self.percore = PersistenceCore()
self.bookie = Bookie()
self.librarian = MemoryLibrarian()
self.oracle = Oracle()
self.postman = MrPostman()
self.salmonator = SalmonatorNoop()
# These are here, for lack of fixturing of the above.
self.doorman = Doorman()
self.enforcer = Enforcer()
self.lawyer = Lawyer()
self.undertaker = Undertaker()
self.rolodex = MockRolodex()
# These are a mix of "necessary" and "unnecessary if well-fixtured"
self.golcore.assemble(self.librarian)
self.ghidproxy.assemble(self.librarian, self.salmonator)
self.oracle.assemble(self.golcore, self.ghidproxy, self.privateer,
self.percore, self.bookie, self.librarian,
self.postman, self.salmonator)
self.privateer.assemble(self.golcore, self.ghidproxy, self.oracle)
self.percore.assemble(self.doorman, self.enforcer, self.lawyer,
self.bookie, self.librarian, self.postman,
self.undertaker, self.salmonator)
self.doorman.assemble(self.librarian)
self.enforcer.assemble(self.librarian)
self.lawyer.assemble(self.librarian)
self.bookie.assemble(self.librarian, self.lawyer, self.undertaker)
self.librarian.assemble(self.percore)
self.postman.assemble(self.golcore, self.librarian, self.bookie,
self.rolodex)
self.undertaker.assemble(self.librarian, self.bookie, self.postman)
# These are both "who-knows-if-necessary-when-fixtured"
credential = MockCredential(self.privateer, TEST_AGENT1)
self.golcore.bootstrap(credential)
self.privateer.prep_bootstrap()
# Just do this manually.
self.privateer._credential = credential
# self.privateer.bootstrap(
# persistent_secrets = {},
# staged_secrets = {},
# chains = {}
# )
self.percore.ingest(TEST_AGENT1.second_party.packed)
def setUp(self):
# Do actually use the fixture, because we need to test the in-memory
# version (plus otherwise, we simply cannot test LibrarianCore).
self.librarian = LibrarianCore.__fixture__()
self.enforcer = Enforcer.__fixture__(self.librarian)
self.lawyer = LawyerCore.__fixture__(self.librarian)
self.percore = PersistenceCore.__fixture__()
# And assemble the librarian
self.librarian.assemble(self.enforcer, self.lawyer, self.percore)
def test_restoration(self):
self.test_trash()
percore = PersistenceCore()
doorman = Doorman()
enforcer = Enforcer()
lawyer = Lawyer()
bookie = Bookie()
librarian = DiskLibrarian(cache_dir='/ghidcache_test')
postman = MrPostman()
undertaker = Undertaker()
salmonator = Salmonator()
rolodex = RolodexMock()
golcore = GCoreMock(TEST_AGENT1)
percore.assemble(doorman, enforcer, lawyer, bookie, librarian, postman,
undertaker, salmonator)
doorman.assemble(librarian)
postman.assemble(golcore, librarian, bookie, rolodex)
undertaker.assemble(librarian, bookie, postman)
lawyer.assemble(librarian)
enforcer.assemble(librarian)
bookie.assemble(librarian, lawyer, undertaker)
librarian.assemble(percore)
salmonator.assemble(percore, percore, doorman, postman, librarian)
# And now reload errything
librarian.restore()
# And make sure the two librarians are equivalent
self.assertEqual(self.librarian._catalog, librarian._catalog)
all_debound = self.bookie._debound_by_ghid.combine(
self.bookie._debound_by_ghid_staged)
all_debound2 = bookie._debound_by_ghid.combine(
bookie._debound_by_ghid_staged)
self.assertEqual(all_debound, all_debound2)
def setUp(self):
''' In addition to the usual, create a tempdir for use.
'''
self.ghidcache = tempfile.mkdtemp()
# Do actually use the fixture, because we need to test the in-memory
# version (plus otherwise, we simply cannot test LibrarianCore).
self.librarian = DiskLibrarian(self.ghidcache, self.executor,
self.nooploop._loop)
self.enforcer = Enforcer.__fixture__(self.librarian)
self.lawyer = LawyerCore.__fixture__(self.librarian)
self.percore = PersistenceCore.__fixture__()
# And assemble the librarian
self.librarian.assemble(self.enforcer, self.lawyer, self.percore)
class DiskLibrarianTrashtest(unittest.TestCase, _GenericPersistenceTest):
def setUp(self):
# Do this on a per-test basis so we have a clean ghidcache for the
# restoration test
clear_ghidcache('/ghidcache_test')
self.percore = PersistenceCore()
self.doorman = Doorman()
self.enforcer = Enforcer()
self.lawyer = Lawyer()
self.bookie = Bookie()
self.librarian = DiskLibrarian(cache_dir='/ghidcache_test')
self.postman = MrPostman()
self.undertaker = Undertaker()
self.salmonator = Salmonator()
self.rolodex = RolodexMock()
self.golcore = GCoreMock(TEST_AGENT1)
self.percore.assemble(self.doorman, self.enforcer, self.lawyer,
self.bookie, self.librarian, self.postman,
self.undertaker, self.salmonator)
self.doorman.assemble(self.librarian)
self.postman.assemble(self.golcore, self.librarian, self.bookie,
self.rolodex)
self.undertaker.assemble(self.librarian, self.bookie, self.postman)
self.lawyer.assemble(self.librarian)
self.enforcer.assemble(self.librarian)
self.bookie.assemble(self.librarian, self.lawyer, self.undertaker)
self.librarian.assemble(self.percore)
self.salmonator.assemble(self.percore, self.percore, self.doorman,
self.postman, self.librarian)
def test_restoration(self):
self.test_trash()
percore = PersistenceCore()
doorman = Doorman()
enforcer = Enforcer()
lawyer = Lawyer()
bookie = Bookie()
librarian = DiskLibrarian(cache_dir='/ghidcache_test')
postman = MrPostman()
undertaker = Undertaker()
salmonator = Salmonator()
rolodex = RolodexMock()
golcore = GCoreMock(TEST_AGENT1)
percore.assemble(doorman, enforcer, lawyer, bookie, librarian, postman,
undertaker, salmonator)
doorman.assemble(librarian)
postman.assemble(golcore, librarian, bookie, rolodex)
undertaker.assemble(librarian, bookie, postman)
lawyer.assemble(librarian)
enforcer.assemble(librarian)
bookie.assemble(librarian, lawyer, undertaker)
librarian.assemble(percore)
salmonator.assemble(percore, percore, doorman, postman, librarian)
# And now reload errything
librarian.restore()
# And make sure the two librarians are equivalent
self.assertEqual(self.librarian._catalog, librarian._catalog)
all_debound = self.bookie._debound_by_ghid.combine(
self.bookie._debound_by_ghid_staged)
all_debound2 = bookie._debound_by_ghid.combine(
bookie._debound_by_ghid_staged)
self.assertEqual(all_debound, all_debound2)
def __init__(self, cache_dir, host, port, *args, **kwargs):
''' Do all of that other smart setup while we're at it.
'''
super().__init__(*args, **kwargs)
self.executor = concurrent.futures.ThreadPoolExecutor()
# Persistence stuff
self.percore = PersistenceCore(self._loop)
self.doorman = Doorman(self.executor, self._loop)
self.enforcer = Enforcer()
self.bookie = Bookie()
self.lawyer = LawyerCore()
self.librarian = DiskLibrarian(cache_dir, self.executor, self._loop)
self.postman = PostOffice()
self.undertaker = UndertakerCore()
# I mean, this won't be used unless we set up peering, but it saves us
# needing to do a modal switch for remote persistence servers
self.salmonator = Salmonator.__fixture__()
self.remote_protocol = RemotePersistenceProtocol()
self.percore.assemble(
doorman = self.doorman,
enforcer = self.enforcer,
lawyer = self.lawyer,
bookie = self.bookie,
librarian = self.librarian,
postman = self.postman,
undertaker = self.undertaker,
salmonator = self.salmonator
)
self.doorman.assemble(librarian=self.librarian)
self.enforcer.assemble(librarian=self.librarian)
self.bookie.assemble(librarian=self.librarian)
self.lawyer.assemble(librarian=self.librarian)
self.librarian.assemble(
enforcer = self.enforcer,
lawyer = self.lawyer,
percore = self.percore
)
self.postman.assemble(
librarian = self.librarian,
remote_protocol = self.remote_protocol
)
self.undertaker.assemble(
librarian = self.librarian,
postman = self.postman
)
self.remote_protocol.assemble(
percore = self.percore,
librarian = self.librarian,
postman = self.postman
)
self.server = BasicServer(connection_cls=WSConnection)
self.register_task(
self.server,
msg_handler = self.remote_protocol,
host = host,
port = port,
tls = False
)
self.register_task(self.postman)
self.register_task(self.undertaker)
class RemotePersistenceServer(loopa.TaskCommander):
''' Simple persistence server.
Expected defaults:
host: 'localhost'
port: 7770
logfile: None
verbosity: 'warning'
debug: False
traceur: False
'''
def __init__(self, cache_dir, host, port, *args, **kwargs):
''' Do all of that other smart setup while we're at it.
'''
super().__init__(*args, **kwargs)
self.executor = concurrent.futures.ThreadPoolExecutor()
# Persistence stuff
self.percore = PersistenceCore(self._loop)
self.doorman = Doorman(self.executor, self._loop)
self.enforcer = Enforcer()
self.bookie = Bookie()
self.lawyer = LawyerCore()
self.librarian = DiskLibrarian(cache_dir, self.executor, self._loop)
self.postman = PostOffice()
self.undertaker = UndertakerCore()
# I mean, this won't be used unless we set up peering, but it saves us
# needing to do a modal switch for remote persistence servers
self.salmonator = Salmonator.__fixture__()
self.remote_protocol = RemotePersistenceProtocol()
self.percore.assemble(
doorman = self.doorman,
enforcer = self.enforcer,
lawyer = self.lawyer,
bookie = self.bookie,
librarian = self.librarian,
postman = self.postman,
undertaker = self.undertaker,
salmonator = self.salmonator
)
self.doorman.assemble(librarian=self.librarian)
self.enforcer.assemble(librarian=self.librarian)
self.bookie.assemble(librarian=self.librarian)
self.lawyer.assemble(librarian=self.librarian)
self.librarian.assemble(
enforcer = self.enforcer,
lawyer = self.lawyer,
percore = self.percore
)
self.postman.assemble(
librarian = self.librarian,
remote_protocol = self.remote_protocol
)
self.undertaker.assemble(
librarian = self.librarian,
postman = self.postman
)
self.remote_protocol.assemble(
percore = self.percore,
librarian = self.librarian,
postman = self.postman
)
self.server = BasicServer(connection_cls=WSConnection)
self.register_task(
self.server,
msg_handler = self.remote_protocol,
host = host,
port = port,
tls = False
)
self.register_task(self.postman)
self.register_task(self.undertaker)
async def setup(self):
''' Once booted, restore the librarian.
'''
await self.librarian.restore()
class OracleTest(unittest.TestCase):
def setUp(self):
# These are directly required by the oracle
self.golcore = GolixCore()
self.ghidproxy = GhidProxier()
self.privateer = Privateer()
self.percore = PersistenceCore()
self.bookie = Bookie()
self.librarian = MemoryLibrarian()
self.oracle = Oracle()
self.postman = MrPostman()
self.salmonator = SalmonatorNoop()
# These are here, for lack of fixturing of the above.
self.doorman = Doorman()
self.enforcer = Enforcer()
self.lawyer = Lawyer()
self.undertaker = Undertaker()
self.rolodex = MockRolodex()
# These are a mix of "necessary" and "unnecessary if well-fixtured"
self.golcore.assemble(self.librarian)
self.ghidproxy.assemble(self.librarian, self.salmonator)
self.oracle.assemble(self.golcore, self.ghidproxy, self.privateer,
self.percore, self.bookie, self.librarian,
self.postman, self.salmonator)
self.privateer.assemble(self.golcore, self.ghidproxy, self.oracle)
self.percore.assemble(self.doorman, self.enforcer, self.lawyer,
self.bookie, self.librarian, self.postman,
self.undertaker, self.salmonator)
self.doorman.assemble(self.librarian)
self.enforcer.assemble(self.librarian)
self.lawyer.assemble(self.librarian)
self.bookie.assemble(self.librarian, self.lawyer, self.undertaker)
self.librarian.assemble(self.percore)
self.postman.assemble(self.golcore, self.librarian, self.bookie,
self.rolodex)
self.undertaker.assemble(self.librarian, self.bookie, self.postman)
# These are both "who-knows-if-necessary-when-fixtured"
credential = MockCredential(self.privateer, TEST_AGENT1)
self.golcore.bootstrap(credential)
self.privateer.prep_bootstrap()
# Just do this manually.
self.privateer._credential = credential
# self.privateer.bootstrap(
# persistent_secrets = {},
# staged_secrets = {},
# chains = {}
# )
self.percore.ingest(TEST_AGENT1.second_party.packed)
def test_load(self):
# First let's try simple retrieval
from _fixtures.remote_exchanges import pt1
from _fixtures.remote_exchanges import cont1_1
from _fixtures.remote_exchanges import secret1_1
geoc1_1 = _GeocLite(cont1_1.ghid, cont1_1.author)
from _fixtures.remote_exchanges import pt2
from _fixtures.remote_exchanges import cont1_2
from _fixtures.remote_exchanges import secret1_2
geoc1_2 = _GeocLite(cont1_2.ghid, cont1_2.author)
from _fixtures.remote_exchanges import dyn1_1a
gobd1_a = _GobdLite(
dyn1_1a.ghid_dynamic,
dyn1_1a.binder,
dyn1_1a.target,
dyn1_1a.ghid,
dyn1_1a.history)
from _fixtures.remote_exchanges import dyn1_1b
gobd1_b = _GobdLite(
dyn1_1b.ghid_dynamic,
dyn1_1b.binder,
dyn1_1b.target,
dyn1_1b.ghid,
dyn1_1b.history)
self.percore.ingest(dyn1_1a.packed)
self.percore.ingest(cont1_1.packed)
self.privateer.stage(cont1_1.ghid, secret1_1)
# Manually call this before next update, to prevent later keyerrors
# from attempting to deliver a "posthumous" notification for the above
# ingestion.
self.postman.do_mail_run()
# Okay, now we should be able to get that.
obj = self.oracle.get_object(_GAO, dyn1_1a.ghid_dynamic)
self.assertEqual(obj.extract_state(), pt1)
# Make sure we loaded it into memory
self.assertIn(dyn1_1a.ghid_dynamic, self.oracle._lookup)
# Test loading it again to get the other code path
obj = self.oracle.get_object(_GAO, dyn1_1a.ghid_dynamic)
# Now, that should have registered with the postman, so let's make sure
# it did, by updating the object.
self.percore.ingest(dyn1_1b.packed)
self.percore.ingest(cont1_2.packed)
self.privateer.stage(cont1_2.ghid, secret1_2)
# Do the mail run and then verify the state has updated
self.postman.do_mail_run()
# Ew. Gross.
time.sleep(.1)
self.assertEqual(obj.extract_state(), pt2)
# TODO: also verify that the object has been registered with the
# salmonator.
def test_new(self):
state = b'\x00'
obj = self.oracle.new_object(_GAO, state, dynamic=True)
# Make sure we called push and therefore have a ghid
self.assertTrue(obj.ghid)
self.assertEqual(obj.extract_state(), state)
# Make sure we loaded it into memory
self.assertIn(obj.ghid, self.oracle._lookup)
class GAOTest(unittest.TestCase):
def setUp(self):
# These are directly required by the GAO
self.golcore = GolixCore()
self.ghidproxy = GhidProxier()
self.privateer = Privateer()
self.percore = PersistenceCore()
self.bookie = Bookie()
self.librarian = MemoryLibrarian()
# These are here, for lack of fixturing of the above.
self.oracle = Oracle()
self.doorman = Doorman()
self.enforcer = Enforcer()
self.lawyer = Lawyer()
self.postman = MrPostman()
self.undertaker = Undertaker()
self.salmonator = SalmonatorNoop()
self.rolodex = MockRolodex()
# These are a mix of "necessary" and "unnecessary if well-fixtured"
self.golcore.assemble(self.librarian)
self.ghidproxy.assemble(self.librarian, self.salmonator)
self.oracle.assemble(self.golcore, self.ghidproxy, self.privateer,
self.percore, self.bookie, self.librarian,
self.postman, self.salmonator)
self.privateer.assemble(self.golcore, self.ghidproxy, self.oracle)
self.percore.assemble(self.doorman, self.enforcer, self.lawyer,
self.bookie, self.librarian, self.postman,
self.undertaker, self.salmonator)
self.doorman.assemble(self.librarian)
self.enforcer.assemble(self.librarian)
self.lawyer.assemble(self.librarian)
self.bookie.assemble(self.librarian, self.lawyer, self.undertaker)
self.librarian.assemble(self.percore)
self.postman.assemble(self.golcore, self.librarian, self.bookie,
self.rolodex)
self.undertaker.assemble(self.librarian, self.bookie, self.postman)
# These are both "who-knows-if-necessary-when-fixtured"
credential = MockCredential(self.privateer, TEST_AGENT1)
self.golcore.bootstrap(credential)
self.privateer.prep_bootstrap()
# Just do this manually.
self.privateer._credential = credential
# self.privateer.bootstrap(
# persistent_secrets = {},
# staged_secrets = {},
# chains = {}
# )
self.percore.ingest(TEST_AGENT1.second_party.packed)
def test_source(self):
''' These tests are alone-ish and can definitely be wholly
fixtured
'''
# Test a static object being created
obj_static = _GAO(self.golcore, self.ghidproxy, self.privateer,
self.percore, self.bookie, self.librarian, False)
msg1 = b'hello stagnant world'
obj_static.apply_state(msg1)
obj_static.push()
self.assertTrue(obj_static.ghid)
self.assertEqual(obj_static.extract_state(), msg1)
self.assertIn(obj_static.ghid, self.privateer)
self.assertIn(obj_static.ghid, self.librarian)
# Now test a dynamic object being created
obj_dyn = _GAO(self.golcore, self.ghidproxy, self.privateer,
self.percore, self.bookie, self.librarian, True)
msg2 = b'hello mutable world'
obj_dyn.apply_state(msg2)
obj_dyn.push()
self.assertTrue(obj_dyn.ghid)
# Speaking of inadequate fixturing...
self.oracle._lookup[obj_dyn.ghid] = obj_dyn
# Back to business as usual now.
self.assertEqual(obj_dyn.extract_state(), msg2)
# We should NOT see the dynamic ghid in privateer.
self.assertNotIn(obj_dyn.ghid, self.privateer)
# But we should see the most recent target
self.assertIn(obj_dyn._history_targets[0], self.privateer)
self.assertIn(obj_dyn.ghid, self.librarian)
# And let's test mutation thereof. 0000001000000110000011100001111
msg3 = b'AFFIRMATIVE'
obj_dyn.apply_state(msg3)
obj_dyn.push()
self.assertTrue(obj_dyn.ghid)
self.assertEqual(obj_dyn.extract_state(), msg3)
# We should NOT see the dynamic ghid in privateer.
self.assertNotIn(obj_dyn.ghid, self.privateer)
# But we should see the most recent target
self.assertIn(obj_dyn._history_targets[0], self.privateer)
self.assertIn(obj_dyn.ghid, self.librarian)
# Now let's try freezing it
frozen_ghid = obj_dyn.freeze()
self.assertEqual(frozen_ghid, obj_dyn._history_targets[0])
# And then delete the original
obj_dyn.delete()
# TODO: assess that this has actually deleted anything...
def test_sink(self):
''' Test retrieval from known ghids, pulls, etc
'''
# "Remote" frame 1
from _fixtures.remote_exchanges import pt1
from _fixtures.remote_exchanges import secret1_1
from _fixtures.remote_exchanges import cont1_1
from _fixtures.remote_exchanges import dyn1_1a
# "Remote" frame 2
from _fixtures.remote_exchanges import pt2
from _fixtures.remote_exchanges import secret1_2
from _fixtures.remote_exchanges import cont1_2
from _fixtures.remote_exchanges import dyn1_1b
self.percore.ingest(dyn1_1a.packed)
self.percore.ingest(cont1_1.packed)
self.privateer.stage(cont1_1.ghid, secret1_1)
obj_dyn = _GAO.from_ghid(dyn1_1a.ghid_dynamic, self.golcore,
self.ghidproxy, self.privateer, self.percore, self.bookie,
self.librarian)
self.assertEqual(obj_dyn.extract_state(), pt1)
# NOTE! These are not ratcheted.
self.percore.ingest(dyn1_1b.packed)
self.percore.ingest(cont1_2.packed)
self.privateer.stage(cont1_2.ghid, secret1_2)
obj_dyn.pull()
self.assertEqual(obj_dyn.extract_state(), pt2)