def test_policy_table_publish(self): """Policy table result publish Test basic DSE functionality with policy engine and table result publish. """ node = helper.make_dsenode_new_partition('testnode') data = fake_datasource.FakeDataSource('data') policy = agnostic.DseRuntime('policy') policy2 = agnostic.DseRuntime('policy2') node.register_service(data) node.register_service(policy) node.register_service(policy2) policy.synchronizer = mock.MagicMock() policy2.synchronizer = mock.MagicMock() policy.create_policy('data', kind=datalog_base.DATASOURCE_POLICY_TYPE) policy.create_policy('classification') policy.set_schema('data', compile.Schema({'q': (1, )})) policy.insert('p(x):-data:q(x),gt(x,2)', target='classification') policy.insert('q(3)', target='data') # TODO(ekcs): test that no publish triggered (because no subscribers) policy2.create_policy('policy') policy2.subscribe('policy', 'classification:p') helper.retry_check_function_return_value( lambda: 'classification:p' in policy. _published_tables_with_subscriber, True) self.assertEqual(list(policy.policySubData.keys()), [('p', 'classification', None)]) helper.retry_check_db_equal(policy2, 'policy:classification:p(x)', 'policy:classification:p(3)') policy.insert('q(4)', target='data') helper.retry_check_db_equal(policy2, 'policy:classification:p(x)', ('policy:classification:p(3)' ' policy:classification:p(4)')) # test that no change to p means no publish triggered policy.insert('q(2)', target='data') # TODO(ekcs): test no publish triggered policy.delete('q(4)', target='data') helper.retry_check_db_equal(policy2, 'policy:classification:p(x)', 'policy:classification:p(3)') policy2.unsubscribe('policy', 'classification:p') # trigger removed helper.retry_check_function_return_value( lambda: len(policy._published_tables_with_subscriber) == 0, True) self.assertEqual(list(policy.policySubData.keys()), []) policy.insert('q(4)', target='data') # TODO(ekcs): test that no publish triggered (because no subscribers) node.stop()
def test_policy_data_late_sub(self): """Test policy correctly processes data on late subscribe.""" node = helper.make_dsenode_new_partition('testnode') data = fake_datasource.FakeDataSource('data') engine = agnostic.DseRuntime(api_base.ENGINE_SERVICE_ID) node.register_service(data) node.register_service(engine) engine.create_policy('policy1') engine.create_policy('data', kind=datalog_base.DATASOURCE_POLICY_TYPE) data.state = {'fake_table': set([(1, ), (2, )])} data.poll() self.insert_rule(engine, 'p(x) :- data:fake_table(x)', 'policy1') helper.retry_check_db_equal(engine, 'p(x)', 'p(1) p(2)', target='policy1') data.state = {'fake_table': set([(1, ), (2, ), (3, )])} data.poll() helper.retry_check_db_equal(engine, 'p(x)', 'p(1) p(2) p(3)', target='policy1') self.assertFalse(hasattr(engine, "last_msg")) node.stop()
def test_policy_data_update(self): """Test policy correctly processes initial data snapshot and update.""" node = helper.make_dsenode_new_partition('testnode') node.always_snapshot = False data = fake_datasource.FakeDataSource('data') engine = agnostic.DseRuntime(api_base.ENGINE_SERVICE_ID) node.register_service(data) node.register_service(engine) engine.create_policy('policy1') engine.create_policy('data') self.insert_rule(engine, 'p(x) :- data:fake_table(x)', 'policy1') data.state = {'fake_table': set([(1, ), (2, )])} data.poll() helper.retry_check_db_equal(engine, 'p(x)', 'p(1) p(2)', target='policy1') data.state = {'fake_table': set([(1, ), (2, ), (3, )])} data.poll() helper.retry_check_db_equal(engine, 'p(x)', 'p(1) p(2) p(3)', target='policy1') self.assertFalse(hasattr(engine, "last_msg")) node.stop()
def test_receive_data_no_sequence_num(self): '''Test receiving data without sequence numbers''' run = agnostic.DseRuntime(api_base.ENGINE_SERVICE_ID) run.always_snapshot = False run.create_policy('datasource1') # initialize with full table run.receive_data_sequenced(publisher='datasource1', table='p', data=[[1], [2]], seqnum=None, is_snapshot=True) actual = run.select('p(x)') correct = 'p(1) p(2)' self.assertTrue(helper.db_equal(actual, correct)) # add data run.receive_data_sequenced(publisher='datasource1', table='p', data=[[[3], [4]], []], seqnum=None, is_snapshot=False) actual = run.select('p(x)') correct = 'p(1) p(2) p(3) p(4)' self.assertTrue(helper.db_equal(actual, correct)) # remove data run.receive_data_sequenced(publisher='datasource1', table='p', data=[[], [[2], [4]]], seqnum=None, is_snapshot=False) actual = run.select('p(x)') correct = 'p(1) p(3)' self.assertTrue(helper.db_equal(actual, correct)) # add & remove data run.receive_data_sequenced(publisher='datasource1', table='p', data=[[[4]], [[3]]], seqnum=None, is_snapshot=False) actual = run.select('p(x)') correct = 'p(1) p(4)' self.assertTrue(helper.db_equal(actual, correct)) # re-initialize with full table run.receive_data_sequenced(publisher='datasource1', table='p', data=[[1], [2]], seqnum=None, is_snapshot=True) actual = run.select('p(x)') correct = 'p(1) p(2)' self.assertTrue(helper.db_equal(actual, correct))
def test_receive_data_arbitrary_start(self): '''Test receiving data with arbitrary starting sequence number''' run = agnostic.DseRuntime(api_base.ENGINE_SERVICE_ID) run.create_policy('datasource1') run.receive_data_sequenced(publisher='datasource1', table='p', data=[[1], [2]], seqnum=1234, is_snapshot=True) actual = run.select('p(x)') correct = 'p(1) p(2)' self.assertTrue(helper.db_equal(actual, correct))
def test_receive_data_out_of_order(self): '''Test receiving data with sequence numbers, out of order''' run = agnostic.DseRuntime(api_base.ENGINE_SERVICE_ID) run.always_snapshot = False run.create_policy('datasource1') # update with lower seqnum than init snapshot is ignored run.receive_data_sequenced(publisher='datasource1', table='p', data=[[[10]], []], seqnum=3, is_snapshot=False) # add & remove data run.receive_data_sequenced(publisher='datasource1', table='p', data=[[[4]], [[3]]], seqnum=7, is_snapshot=False) actual = run.select('p(x)') correct = '' self.assertTrue(helper.db_equal(actual, correct)) # remove data run.receive_data_sequenced(publisher='datasource1', table='p', data=[[], [[2], [4]]], seqnum=6, is_snapshot=False) actual = run.select('p(x)') correct = '' self.assertTrue(helper.db_equal(actual, correct)) # add data run.receive_data_sequenced(publisher='datasource1', table='p', data=[[[3], [4]], []], seqnum=5, is_snapshot=False) actual = run.select('p(x)') correct = '' self.assertTrue(helper.db_equal(actual, correct)) # initialize with full table run.receive_data_sequenced(publisher='datasource1', table='p', data=[[1], [2]], seqnum=4, is_snapshot=True) actual = run.select('p(x)') correct = 'p(1) p(4)' self.assertTrue(helper.db_equal(actual, correct))
def test_receive_data_sequence_number_max_int(self): '''Test receiving data when sequence number goes over max int''' run = agnostic.DseRuntime(api_base.ENGINE_SERVICE_ID) run.always_snapshot = False run.create_policy('datasource1') run.receive_data_sequenced(publisher='datasource1', table='p', data=[[1], [2]], seqnum=sys.maxsize, is_snapshot=True) actual = run.select('p(x)') correct = 'p(1) p(2)' self.assertTrue(helper.db_equal(actual, correct)) run.receive_data_sequenced(publisher='datasource1', table='p', data=[[], [[2]]], seqnum=sys.maxsize + 1, is_snapshot=False) actual = run.select('p(x)') correct = 'p(1)' self.assertTrue(helper.db_equal(actual, correct)) # test out-of-sequence update ignored run.receive_data_sequenced(publisher='datasource1', table='p', data=[[[2]], []], seqnum=sys.maxsize, is_snapshot=False) actual = run.select('p(x)') correct = 'p(1)' self.assertTrue(helper.db_equal(actual, correct)) run.receive_data_sequenced(publisher='datasource1', table='p', data=[[[4]], []], seqnum=sys.maxsize + 3, is_snapshot=False) actual = run.select('p(x)') correct = 'p(1)' self.assertTrue(helper.db_equal(actual, correct)) run.receive_data_sequenced(publisher='datasource1', table='p', data=[[[3]], []], seqnum=sys.maxsize + 2, is_snapshot=False) actual = run.select('p(x)') correct = 'p(1) p(3) p(4)' self.assertTrue(helper.db_equal(actual, correct))
def test_receive_data_duplicate_sequence_number(self): '''Test receiving data with duplicate sequence number Only one message (arbitrary) should be processed. ''' run = agnostic.DseRuntime(api_base.ENGINE_SERVICE_ID) run.always_snapshot = False run.create_policy('datasource1') # send three updates with the same seqnum run.receive_data_sequenced(publisher='datasource1', table='p', data=[[[1]], []], seqnum=1, is_snapshot=False) run.receive_data_sequenced(publisher='datasource1', table='p', data=[[[2]], []], seqnum=1, is_snapshot=False) run.receive_data_sequenced(publisher='datasource1', table='p', data=[[[3]], []], seqnum=1, is_snapshot=False) # start with empty data run.receive_data_sequenced(publisher='datasource1', table='p', data=[], seqnum=0, is_snapshot=True) # exactly one of the three updates should be applied actual = run.select('p(x)') correct1 = 'p(1)' correct2 = 'p(2)' correct3 = 'p(3)' self.assertTrue( helper.db_equal(actual, correct1) or helper.db_equal(actual, correct2) or helper.db_equal(actual, correct3))
def create_policy_engine(): """Create policy engine and initialize it using the api models.""" engine = agnostic.DseRuntime(api_base.ENGINE_SERVICE_ID) engine.debug_mode() # should take this out for production return engine
def test_replicated_pe_exec(self): """Test correct local leader behavior with 2 PEs requesting exec""" node1 = helper.make_dsenode_new_partition('testnode1') node2 = helper.make_dsenode_same_partition(node1, 'testnode2') dsd = fake_datasource.FakeDataSource('dsd') # faster time-out for testing dsd.LEADER_TIMEOUT = 2 pe1 = agnostic.DseRuntime('pe1') pe2 = agnostic.DseRuntime('pe2') node1.register_service(pe1) node2.register_service(pe2) node1.register_service(dsd) assert dsd._running assert node1._running assert node2._running assert node1._control_bus._running # first exec request obeyed and leader set pe2.rpc( 'dsd', 'request_execute', { 'action': 'fake_act', 'action_args': { 'name': 'testnode2' }, 'wait': True }) helper.retry_check_function_return_value(lambda: len(dsd.exec_history), 1) self.assertEqual(dsd._leader_node_id, 'testnode2') # second exec request from leader obeyed and leader remains pe2.rpc( 'dsd', 'request_execute', { 'action': 'fake_act', 'action_args': { 'name': 'testnode2' }, 'wait': True }) helper.retry_check_function_return_value(lambda: len(dsd.exec_history), 2) self.assertEqual(dsd._leader_node_id, 'testnode2') # exec request from non-leader not obeyed pe1.rpc( 'dsd', 'request_execute', { 'action': 'fake_act', 'action_args': { 'name': 'testnode1' }, 'wait': True }) self.assertRaises(tenacity.RetryError, helper.retry_check_function_return_value, lambda: len(dsd.exec_history), 3) # leader vacated after heartbeat stops node2.stop() node2.wait() helper.retry_check_function_return_value(lambda: dsd._leader_node_id, None) # next exec request obeyed and new leader set pe1.rpc( 'dsd', 'request_execute', { 'action': 'fake_act', 'action_args': { 'name': 'testnode1' }, 'wait': True }) helper.retry_check_function_return_value(lambda: len(dsd.exec_history), 3) self.assertEqual(dsd._leader_node_id, 'testnode1') node1.stop() node2.stop()
def test_receive_data_multiple_tables(self): '''Test receiving data with sequence numbers, multiple tables''' run = agnostic.DseRuntime(api_base.ENGINE_SERVICE_ID) run.create_policy('datasource1') # initialize p with full table run.receive_data_sequenced(publisher='datasource1', table='p', data=[[1]], seqnum=0, is_snapshot=True) actual = run.select('p(x)') correct = 'p(1)' self.assertTrue(helper.db_equal(actual, correct)) # add data to p run.receive_data_sequenced(publisher='datasource1', table='p', data=[[[2]], []], seqnum=1, is_snapshot=False) actual = run.select('p(x)') correct = 'p(1) p(2)' self.assertTrue(helper.db_equal(actual, correct)) # add data to q run.receive_data_sequenced(publisher='datasource1', table='q', data=[[[2]], []], seqnum=1, is_snapshot=False) actual = run.select('q(x)') correct = '' # does not apply until initialize self.assertTrue(helper.db_equal(actual, correct)) # initialize q with full table run.receive_data_sequenced(publisher='datasource1', table='q', data=[[1]], seqnum=0, is_snapshot=True) actual = run.select('q(x)') correct = 'q(1) q(2)' # both initial data and preceding update applied self.assertTrue(helper.db_equal(actual, correct)) # add data to q run.receive_data_sequenced(publisher='datasource1', table='q', data=[[[3]], []], seqnum=2, is_snapshot=False) actual = run.select('q(x)') correct = 'q(1) q(2) q(3)' self.assertTrue(helper.db_equal(actual, correct)) # add data to p run.receive_data_sequenced(publisher='datasource1', table='p', data=[[[3]], []], seqnum=2, is_snapshot=False) actual = run.select('p(x)') correct = 'p(1) p(2) p(3)' self.assertTrue(helper.db_equal(actual, correct))