def setUp(self): spec1 = TestSpec1() spec2 = TestSpec2() self.local = VoodooStore(attrs=['push1','push2','pull','key','share','id'], key='id', push_attrs=['pull'], keygen=lambda dobj: 9, pushgen=lambda dobj: 10) self.local1 = FacetStore(self.local) self.api1 = VoodooStore(spec=spec1, pushgen=lambda dobj: 11) self.snapshot1 = VoodooStore(spec=spec1) self.local2 = FacetStore(self.local) self.api2 = VoodooStore(spec=spec2, pushgen=lambda dobj: 12) self.snapshot2 = VoodooStore(spec=spec2) self.arm1 = Arm(spec1, self.api1,self.local1,self.snapshot1) self.arm2 = Arm(spec2, self.api2,self.local2,self.snapshot2) self.log = logging_helper.get_log('cynq.test')
def test_one_arm_pushkey_local_create_with_nonekey(self): class PushKeySpec(BaseSpec): name = 'pushkey' rpushed = ('key',) rpulled = ('pull',) shared = ('share',) key = 'key' spec = PushKeySpec() local = VoodooStore(attrs=['key','pull','share','id'], key='id', push_attrs=['pull'], keygen=lambda dobj: 9, pushgen=lambda dobj: 10) api = VoodooStore(spec=spec, keygen=lambda dobj: 11) snapshot = VoodooStore(spec=spec) arm = Arm(spec, api, local, snapshot) local.ddata = [{'id':8, 'share':4, 'pull':3, 'key':None}] expectedL = [{'id':8, 'pull':3, 'share':4, 'key':11}] expectedR = [{'key':11, 'pull':3, 'share':4}] cynq = Controller(arm).cynq() self.assert_store(expectedL,(1,0,1,0),local) self.assert_store(expectedR,(1,1,0,0),api) self.assert_store(expectedR,(1,1,0,0),snapshot) self.assert_no_changes_on_another_cynq(cynq, local)
class FullTestCase(TestCase): def setUp(self): spec1 = TestSpec1() spec2 = TestSpec2() self.local = VoodooStore(attrs=['push1','push2','pull','key','share','id'], key='id', push_attrs=['pull'], keygen=lambda dobj: 9, pushgen=lambda dobj: 10) self.local1 = FacetStore(self.local) self.api1 = VoodooStore(spec=spec1, pushgen=lambda dobj: 11) self.snapshot1 = VoodooStore(spec=spec1) self.local2 = FacetStore(self.local) self.api2 = VoodooStore(spec=spec2, pushgen=lambda dobj: 12) self.snapshot2 = VoodooStore(spec=spec2) self.arm1 = Arm(spec1, self.api1,self.local1,self.snapshot1) self.arm2 = Arm(spec2, self.api2,self.local2,self.snapshot2) self.log = logging_helper.get_log('cynq.test') def tearDown(self): pass def test_one_arm_remote_create(self): self.api1.ddata = [{'key':1, 'share':4, 'pull':3, 'push1':2}] expectedL = [{'id':9, 'key':1, 'share':4, 'pull':10, 'push1':2, 'push2':None}] expected1 = [{'key':1, 'share':4, 'pull':10, 'push1':2}] cynq = Controller(self.arm1).cynq() self.assert_store(expectedL,(1,1,0,0),self.local) self.assert_store(expected1,(1,0,1,0),self.api1) self.assert_store(expected1,(1,1,0,0),self.snapshot1) self.assert_no_changes_on_another_cynq(cynq) def test_two_arm_remote_create(self): self.api1.ddata = [{'key':1, 'share':4, 'pull':3, 'push1':2}] expectedL = [{'id':9, 'key':1, 'share':4, 'pull':10, 'push1':2, 'push2':12}] expected1 = [{'key':1, 'share':4, 'pull':10, 'push1':2}] expected2 = [{'key':1, 'share':4, 'pull':10, 'push2':12}] cynq = Controller(self.arm1, self.arm2).cynq() self.assert_store(expectedL,(1,1,1,0),self.local) self.assert_store(expected1,(1,0,1,0),self.api1) self.assert_store(expected1,(1,1,0,0),self.snapshot1) self.assert_store(expected2,(1,1,0,0),self.api2) self.assert_store(expected2,(1,1,0,0),self.snapshot2) self.assert_no_changes_on_another_cynq(cynq) def test_one_arm_local_create(self): self.local.ddata = [{'id':5, 'key':1, 'share':4, 'pull':3, 'push1':2, 'push2':None}] expectedL = [{'id':5, 'key':1, 'share':4, 'pull':3, 'push1':11, 'push2':None}] expected1 = [{'key':1, 'share':4, 'pull':3, 'push1':11}] cynq = Controller(self.arm1).cynq() self.assert_store(expectedL,(1,0,1,0),self.local) self.assert_store(expected1,(1,1,0,0),self.api1) self.assert_store(expected1,(1,1,0,0),self.snapshot1) self.assert_no_changes_on_another_cynq(cynq) def test_two_arm_local_create(self): self.local.ddata = [{'id':5, 'key':1, 'share':4, 'pull':3, 'push1':2, 'push2':None}] expectedL = [{'id':5, 'key':1, 'share':4, 'pull':3, 'push1':11, 'push2':12}] expected1 = [{'key':1, 'share':4, 'pull':3, 'push1':11}] expected2 = [{'key':1, 'share':4, 'pull':3, 'push2':12}] cynq = Controller(self.arm1, self.arm2).cynq() self.assert_store(expectedL,(1,0,2,0),self.local) self.assert_store(expected1,(1,1,0,0),self.api1) self.assert_store(expected1,(1,1,0,0),self.snapshot1) self.assert_store(expected2,(1,1,0,0),self.api2) self.assert_store(expected2,(1,1,0,0),self.snapshot2) self.assert_no_changes_on_another_cynq(cynq) def test_one_arm_local_delete(self): seeds = [{'key':1, 'share':4, 'pull':3, 'push1':2}] self.api1.ddata = seeds self.snapshot1.ddata = seeds expected = [] cynq = Controller(self.arm1).cynq() self.assert_store(expected,(1,0,0,0),self.local) self.assert_store(expected,(1,0,0,1),self.api1) self.assert_store(expected,(1,0,0,1),self.snapshot1) self.assert_no_changes_on_another_cynq(cynq) def test_two_arm_local_delete(self): seeds = [{'key':1, 'share':4, 'pull':3, 'push1':2}] self.api1.ddata = seeds self.snapshot1.ddata = seeds self.api2.ddata = seeds self.snapshot2.ddata = seeds expected = [] cynq = Controller(self.arm1, self.arm2).cynq() self.assert_store(expected,(1,0,0,0),self.local) self.assert_store(expected,(1,0,0,1),self.api1) self.assert_store(expected,(1,0,0,1),self.snapshot1) self.assert_store(expected,(1,0,0,1),self.api2) self.assert_store(expected,(1,0,0,1),self.snapshot2) self.assert_no_changes_on_another_cynq(cynq) def test_one_arm_remote_delete(self): self.local.ddata = [{'id':5, 'key':1, 'share':4, 'pull':3, 'push1':2, 'push2':None}] self.snapshot1.ddata = [{'key':1, 'share':4, 'pull':3, 'push1':2}] expected = [] cynq = Controller(self.arm1).cynq() self.assert_store(expected,(1,0,0,1),self.local) self.assert_store(expected,(1,0,0,0),self.api1) self.assert_store(expected,(1,0,0,1),self.snapshot1) self.assert_no_changes_on_another_cynq(cynq) def test_two_arm_remote_delete(self): self.local.ddata = [{'id':5, 'key':1, 'share':4, 'pull':3, 'push1':2, 'push2':None}] seeds = [{'key':1, 'share':4, 'pull':3, 'push1':2}] self.snapshot1.ddata = seeds self.snapshot2.ddata = seeds self.api2.ddata = seeds expected = [] cynq = Controller(self.arm1, self.arm2).cynq() self.assert_store(expected,(1,0,0,1),self.local) self.assert_store(expected,(1,0,0,0),self.api1) self.assert_store(expected,(1,0,0,1),self.snapshot1) self.assert_store(expected,(1,0,0,1),self.api2) self.assert_store(expected,(1,0,0,1),self.snapshot2) self.assert_no_changes_on_another_cynq(cynq) def test_one_arm_local_update(self): self.local.ddata = [{'id':5, 'key':1, 'share':6, 'pull':3, 'push1':2, 'push2':None}] seeds = [{'key':1, 'share':4, 'pull':3, 'push1':2}] self.api1.ddata = seeds self.snapshot1.ddata = seeds expectedL = [{'id':5, 'key':1, 'share':6, 'pull':3, 'push1':2, 'push2':None}] expected1 = [{'key':1, 'share':6, 'pull':3, 'push1':2}] cynq = Controller(self.arm1).cynq() self.assert_store(expectedL,(1,0,0,0),self.local) self.assert_store(expected1,(1,0,1,0),self.api1) self.assert_store(expected1,(1,0,1,0),self.snapshot1) self.assert_no_changes_on_another_cynq(cynq) def test_two_arm_local_update(self): seeds1 = [{'key':1, 'share':4, 'pull':3, 'push1':2}] seeds2 = [{'key':1, 'share':4, 'pull':3, 'push2':None}] self.api1.ddata = seeds1 self.snapshot1.ddata = seeds1 self.api2.ddata = seeds2 self.snapshot2.ddata = seeds2 expectedL = [{'id':5, 'key':1, 'share':6, 'pull':3, 'push1':2, 'push2':None}] expected1 = [{'key':1, 'share':6, 'pull':3, 'push1':2}] expected2 = [{'key':1, 'share':6, 'pull':3, 'push2':None}] self.local.ddata = expectedL cynq = Controller(self.arm1,self.arm2).cynq() self.assert_store(expectedL,(1,0,0,0),self.local) self.assert_store(expected1,(1,0,1,0),self.api1) self.assert_store(expected1,(1,0,1,0),self.snapshot1) self.assert_store(expected2,(1,0,1,0),self.api2) self.assert_store(expected2,(1,0,1,0),self.snapshot2) self.assert_no_changes_on_another_cynq(cynq) def test_one_arm_remote_update(self): self.local.ddata = [{'id':5, 'key':1, 'share':4, 'pull':3, 'push1':2, 'push2':None}] self.snapshot1.ddata = [{'key':1, 'share':4, 'pull':3, 'push1':2}] expectedL = [{'id':5, 'key':1, 'share':6, 'pull':3, 'push1':2, 'push2':None}] expected1 = [{'key':1, 'share':6, 'pull':3, 'push1':2}] self.api1.ddata = expected1 cynq = Controller(self.arm1).cynq() self.assert_store(expectedL,(1,0,1,0),self.local) self.assert_store(expected1,(1,0,0,0),self.api1) self.assert_store(expected1,(1,0,1,0),self.snapshot1) self.assert_no_changes_on_another_cynq(cynq) def test_two_arm_remote_update(self): self.local.ddata = [{'id':5, 'key':1, 'share':4, 'pull':3, 'push1':2, 'push2':None}] self.snapshot1.ddata = [{'key':1, 'share':4, 'pull':3, 'push1':2}] self.snapshot2.ddata = [{'key':1, 'share':4, 'pull':3, 'push2':None}] self.api1.ddata = [{'key':1, 'share':4, 'pull':3, 'push1':2}] self.api2.ddata = [{'key':1, 'share':6, 'pull':3, 'push2':None}] expectedL = [{'id':5, 'key':1, 'share':6, 'pull':3, 'push1':2, 'push2':None}] expected1 = [{'key':1, 'share':6, 'pull':3, 'push1':2}] expected2 = [{'key':1, 'share':6, 'pull':3, 'push2':None}] cynq = Controller(self.arm1,self.arm2).cynq() self.assert_store(expectedL,(1,0,1,0),self.local) self.assert_store(expected1,(1,0,1,0),self.api1) self.assert_store(expected1,(1,0,1,0),self.snapshot1) self.assert_store(expected2,(1,0,0,0),self.api2) self.assert_store(expected2,(1,0,1,0),self.snapshot2) self.assert_no_changes_on_another_cynq(cynq) def test_two_arm_nothing_test(self): self.local.ddata = [{'id':5, 'key':1, 'share':6, 'pull':3, 'push1':2, 'push2':None}] self.snapshot1.ddata = [{'key':1, 'share':6, 'pull':3, 'push1':2}] self.snapshot2.ddata = [{'key':1, 'share':6, 'pull':3, 'push2':None}] self.api1.ddata = [{'key':1, 'share':6, 'pull':3, 'push1':2}] self.api2.ddata = [{'key':1, 'share':6, 'pull':3, 'push2':None}] expectedL = [{'id':5, 'key':1, 'share':6, 'pull':3, 'push1':2, 'push2':None}] expected1 = [{'key':1, 'share':6, 'pull':3, 'push1':2}] expected2 = [{'key':1, 'share':6, 'pull':3, 'push2':None}] cynq = Controller(self.arm1,self.arm2).cynq() self.assert_store(expectedL,(1,0,0,0),self.local) self.assert_store(expected1,(1,0,0,0),self.api1) self.assert_store(expected1,(1,0,0,0),self.snapshot1) self.assert_store(expected2,(1,0,0,0),self.api2) self.assert_store(expected2,(1,0,0,0),self.snapshot2) self.assert_no_changes_on_another_cynq(cynq) def test_two_arm_double_test(self): self.local.ddata = [{'id':5, 'key':1, 'share':6, 'pull':3, 'push1':2, 'push2':None}] self.snapshot1.ddata = [{'key':1, 'share':6, 'pull':3, 'push1':2}] self.snapshot2.ddata = [{'key':1, 'share':6, 'pull':3, 'push2':None}] self.api1.ddata = [{'key':1, 'share':6, 'pull':3, 'push1':2}, {'key':1, 'share':6, 'pull':3, 'push1':2} ] self.api2.ddata = [{'key':1, 'share':6, 'pull':3, 'push2':None}] with self.assertRaises(Error): Controller(self.arm1,self.arm2).cynq() def test_snapshot_recovery(self): expectedL = [{'id':5, 'key':1, 'share':4, 'pull':3, 'push1':2, 'push2':None}] expected1 = [{'key':1, 'share':4, 'pull':3, 'push1':2}] self.local.ddata = expectedL self.api1.ddata = expected1 cynq = Controller(self.arm1).cynq() self.assert_store(expectedL,(1,0,0,0),self.local) self.assert_store(expected1,(1,0,0,0),self.api1) self.assert_store(expected1,(1,1,0,0),self.snapshot1) self.assert_no_changes_on_another_cynq(cynq) def test_initial_setup(self): expectedL = [{'id':9, 'key':1, 'share':4, 'pull':10, 'push1':2, 'push2':None}] expected1 = [{'key':1, 'share':4, 'pull':10, 'push1':2}] self.api1.ddata = expected1 cynq = Controller(self.arm1).cynq() self.assert_store(expectedL,(1,1,0,0),self.local) self.assert_store(expected1,(1,0,0,0),self.api1) self.assert_store(expected1,(1,1,0,0),self.snapshot1) self.assert_no_changes_on_another_cynq(cynq) def test_one_arm_mixed_update(self): self.local.ddata = [{'id':5, 'key':1, 'share':7, 'pull':6, 'push1':2, 'push2':None}] self.snapshot1.ddata = [{'key':1, 'share':4, 'pull':3, 'push1':2}] self.api1.ddata = [{'key':1, 'share':8, 'pull':3, 'push1':5}] expectedL = [{'id':5, 'key':1, 'share':7, 'pull':6, 'push1':5, 'push2':None}] expected1 = [{'key':1, 'share':7, 'pull':6, 'push1':5}] cynq = Controller(self.arm1).cynq() self.assert_store(expectedL,(1,0,1,0),self.local) self.assert_store(expected1,(1,0,1,0),self.api1) self.assert_store(expected1,(1,0,1,0),self.snapshot1) self.assert_no_changes_on_another_cynq(cynq) def test_one_arm_pushkey_local_create(self): class PushKeySpec(BaseSpec): name = 'pushkey' rpushed = ('key',) rpulled = ('pull',) shared = ('share',) key = 'key' spec = PushKeySpec() local = VoodooStore(attrs=['key','pull','share','id'], key='id', push_attrs=['pull'], keygen=lambda dobj: 9, pushgen=lambda dobj: 10) api = VoodooStore(spec=spec, keygen=lambda dobj: 11) snapshot = VoodooStore(spec=spec) arm = Arm(spec, api, local, snapshot) local.ddata = [{'id':8, 'share':4, 'pull':3}] expectedL = [{'id':8, 'pull':3, 'share':4, 'key':11}] expectedR = [{'key':11, 'pull':3, 'share':4}] cynq = Controller(arm).cynq() self.assert_store(expectedL,(1,0,1,0),local) self.assert_store(expectedR,(1,1,0,0),api) self.assert_store(expectedR,(1,1,0,0),snapshot) self.assert_no_changes_on_another_cynq(cynq, local) def test_one_arm_pushkey_local_create_with_nonekey(self): class PushKeySpec(BaseSpec): name = 'pushkey' rpushed = ('key',) rpulled = ('pull',) shared = ('share',) key = 'key' spec = PushKeySpec() local = VoodooStore(attrs=['key','pull','share','id'], key='id', push_attrs=['pull'], keygen=lambda dobj: 9, pushgen=lambda dobj: 10) api = VoodooStore(spec=spec, keygen=lambda dobj: 11) snapshot = VoodooStore(spec=spec) arm = Arm(spec, api, local, snapshot) local.ddata = [{'id':8, 'share':4, 'pull':3, 'key':None}] expectedL = [{'id':8, 'pull':3, 'share':4, 'key':11}] expectedR = [{'key':11, 'pull':3, 'share':4}] cynq = Controller(arm).cynq() self.assert_store(expectedL,(1,0,1,0),local) self.assert_store(expectedR,(1,1,0,0),api) self.assert_store(expectedR,(1,1,0,0),snapshot) self.assert_no_changes_on_another_cynq(cynq, local) def test_two_arm_large_cynq(self): self.assertTrue(True) return self.local.generate_seeds(1000) self.api1.generate_seeds(1000) self.api2.generate_seeds(1000) cynq = Controller(self.arm1,self.arm2).cynq() self.log.debug("local changes: %s", self.local.post_stats) self.log.debug("api1 changes: %s", self.api1.post_stats) self.log.debug("api2 changes: %s", self.api2.post_stats) self.log.debug("snapshot1 changes: %s", self.snapshot1.post_stats) self.log.debug("snapshot2 changes: %s", self.snapshot2.post_stats) self.assert_no_changes_on_another_cynq(cynq) def test_two_arm_large_cynq_with_duplicate(self): self.assertTrue(True) return self.local.generate_seeds(1000) self.api1.generate_seeds(1000) self.api2.generate_seeds(1000) cynq = Controller(self.arm1,self.arm2).cynq() self.log.debug("local changes: %s", self.local.post_stats) self.log.debug("api1 changes: %s", self.api1.post_stats) self.log.debug("api2 changes: %s", self.api2.post_stats) self.log.debug("snapshot1 changes: %s", self.snapshot1.post_stats) self.log.debug("snapshot2 changes: %s", self.snapshot2.post_stats) self.assert_no_changes_on_another_cynq(cynq) def test_one_arm_translated_remote_create(self): self.api1.attrs = set(['keyx','sharex','pullx','push1x']) self.api1.vkey = 'keyx' self.api1.push_attrs = set(['push1x']) self.api1.translation = {'key':'keyx', 'share':'sharex', 'pull':'pullx', 'push1':'push1x'} # self.api1.ddata = [{'key':1, 'share':4, 'pull':3, 'push1':2}] self.api1.ddata = [{'keyx':1, 'sharex':4, 'pullx':3, 'push1x':2}] expectedL = [{'id':9, 'key':1, 'share':4, 'pull':10, 'push1':2, 'push2':None}] expectedS = [{'key':1, 'share':4, 'pull':10, 'push1':2}] expected1 = [{'keyx':1, 'sharex':4, 'pullx':10, 'push1x':2}] #expected1 = [{'key':1, 'share':4, 'pull':10, 'push1':2}] cynq = Controller(self.arm1).cynq() self.assert_store(expectedL,(1,1,0,0),self.local) self.assert_store(expectedS,(1,0,1,0),self.api1) self.assert_store(expectedS,(1,1,0,0),self.snapshot1) self.assert_equal_dlists(self.api1.attrs, expected1, self.api1.ddata) self.assert_no_changes_on_another_cynq(cynq) def test_falsy_one_arm_local_update_to_none(self): self.local.ddata = [{'id':5, 'key':1, 'share':None, 'pull':3, 'push1':2, 'push2':None}] seeds = [{'key':1, 'share':False, 'pull':3, 'push1':2}] self.api1.ddata = seeds self.snapshot1.ddata = seeds expectedL = [{'id':5, 'key':1, 'share':None, 'pull':3, 'push1':2, 'push2':None}] expected1 = [{'key':1, 'share':None, 'pull':3, 'push1':2}] cynq = Controller(self.arm1).cynq() self.assert_store(expectedL,(1,0,0,0),self.local) self.assert_store(expected1,(1,0,1,0),self.api1) self.assert_store(expected1,(1,0,1,0),self.snapshot1) self.assert_no_changes_on_another_cynq(cynq) def test_falsy_one_arm_local_update_to_false(self): self.local.ddata = [{'id':5, 'key':1, 'share':False, 'pull':3, 'push1':2, 'push2':None}] seeds = [{'key':1, 'share':None, 'pull':3, 'push1':2}] self.api1.ddata = seeds self.snapshot1.ddata = seeds expectedL = [{'id':5, 'key':1, 'share':False, 'pull':3, 'push1':2, 'push2':None}] expected1 = [{'key':1, 'share':False, 'pull':3, 'push1':2}] cynq = Controller(self.arm1).cynq() self.assert_store(expectedL,(1,0,0,0),self.local) self.assert_store(expected1,(1,0,1,0),self.api1) self.assert_store(expected1,(1,0,1,0),self.snapshot1) self.assert_no_changes_on_another_cynq(cynq) def assert_no_changes_on_another_cynq(self, controller, local=None): if not local: local = self.local local._clear_cache(); local._clear_stats() for arm in controller.arms: for s in arm.stores: s._clear_cache(); if s != arm.local: s._clear_stats() # make sure it's all equal to start arm_ddata = {} for arm in controller.arms: self.assert_equal_dlists(arm.spec.attrs_with_key, arm.api.backward_translated_ddata, local.backward_translated_ddata) self.assert_equal_dlists(arm.spec.attrs_with_key, arm.api.backward_translated_ddata, arm.snapshot.backward_translated_ddata) self.assert_equal_dlists(arm.spec.attrs_with_key, local.backward_translated_ddata, arm.snapshot.backward_translated_ddata) # overkill just in case arm_ddata[arm] = arm.api.backward_translated_ddata local_ddata = local.ddata controller.cynq() for arm in controller.arms: self.assert_store(local_ddata,(1,0,0,0),local) self.assert_store(arm_ddata[arm],(1,0,0,0),arm.api) self.assert_store(arm_ddata[arm],(1,0,0,0),arm.snapshot) def assert_store(self, *args): args = list(args) if len(args)==3: args[2:2]=[args[1]] # if no post/pre disctinction, then just double up expected_ddata, expected_pre_stats, expected_post_stats, store = args self.assertEqual(expected_pre_stats, store.pre_stats) self.assertEqual(expected_post_stats, store.post_stats) self.assert_equal_dlists(store.attrs, expected_ddata, store.backward_translated_ddata)