def test_policy_data_update(self):
"""Test policy correctly processes initial data snapshot and update."""
node = helper.make_dsenode_new_partition('testnode')
node.always_snapshot = False
data = fake_datasource.FakeDataSource('data')
engine = agnostic.DseRuntime(api_base.ENGINE_SERVICE_ID)
node.register_service(data)
node.register_service(engine)
engine.create_policy('policy1')
engine.create_policy('data')
self.insert_rule(engine, 'p(x) :- data:fake_table(x)', 'policy1')
data.state = {'fake_table': set([(1, ), (2, )])}
data.poll()
helper.retry_check_db_equal(engine,
'p(x)',
'p(1) p(2)',
target='policy1')
data.state = {'fake_table': set([(1, ), (2, ), (3, )])}
data.poll()
helper.retry_check_db_equal(engine,
'p(x)',
'p(1) p(2) p(3)',
target='policy1')
self.assertFalse(hasattr(engine, "last_msg"))
node.stop()
def test_policy_data_late_sub(self):
"""Test policy correctly processes data on late subscribe."""
node = helper.make_dsenode_new_partition('testnode')
data = fake_datasource.FakeDataSource('data')
engine = agnostic.DseRuntime(api_base.ENGINE_SERVICE_ID)
node.register_service(data)
node.register_service(engine)
engine.create_policy('policy1')
engine.create_policy('data', kind=datalog_base.DATASOURCE_POLICY_TYPE)
data.state = {'fake_table': set([(1, ), (2, )])}
data.poll()
self.insert_rule(engine, 'p(x) :- data:fake_table(x)', 'policy1')
helper.retry_check_db_equal(engine,
'p(x)',
'p(1) p(2)',
target='policy1')
data.state = {'fake_table': set([(1, ), (2, ), (3, )])}
data.poll()
helper.retry_check_db_equal(engine,
'p(x)',
'p(1) p(2) p(3)',
target='policy1')
self.assertFalse(hasattr(engine, "last_msg"))
node.stop()
def test_policy_data(self):
"""Test policy properly inserts data and processes it normally."""
cage = congress.dse.d6cage.d6Cage()
cage.loadModule(
"TestDriver",
helper.data_module_path("../tests/datasources/test_driver.py"))
cage.loadModule("TestPolicy", helper.policy_module_path())
cage.createservice(name="data",
moduleName="TestDriver",
args=helper.datasource_openstack_args())
cage.createservice(name="policy",
moduleName="TestPolicy",
args={
'd6cage': cage,
'rootdir': ''
})
data = cage.services['data']['object']
policy = cage.services['policy']['object']
# turn off module-schema syntax checking
policy.create_policy('data')
policy.set_schema('data', compile.Schema({'p': (1, )}))
policy.subscribe('data', 'p', callback=policy.receive_data)
formula = policy.parse1('p(1)')
# sending a single Insert. (Default for Event is Insert.)
data.publish('p', [runtime.Event(formula)])
helper.retry_check_db_equal(policy, 'data:p(x)', 'data:p(1)')
def test_set_policy(self):
LOG.info("set_policy")
self.vmplace.set_policy('p(x) :- fake:q(x)')
self.fake.state = {'q': set([tuple([1]), tuple([2])])}
self.fake.poll()
ans = ('p(1) p(2)')
helper.retry_check_db_equal(self.vmplace, 'p(x)', ans)
def test_set_policy(self):
LOG.info("set_policy")
self.vmplace.set_policy('p(x) :- fake:q(x)')
self.fake.state = {'q': set([tuple([1]), tuple([2])])}
self.fake.poll()
ans = ('p(1) p(2)')
helper.retry_check_db_equal(self.vmplace, 'p(x)', ans)
def test_set_policy(self):
LOG.info("set_policy")
self.vmplace.set_policy("p(x) :- fake:q(x)")
self.fake.state = {"q": set([tuple([1]), tuple([2])])}
self.fake.poll()
ans = "p(1) p(2)"
helper.retry_check_db_equal(self.vmplace, "p(x)", ans)
def test_rule_api_model(self):
"""Test the rule api model.
Same as test_multiple except we use the api interface
instead of the DSE interface.
"""
api = self.api
cage = self.cage
engine = self.engine
policy = engine.DEFAULT_THEORY
# Insert formula
net_formula = test_neutron.create_networkXnetwork_group('p')
LOG.debug("Sending formula: %s", net_formula)
engine.debug_mode()
context = {'policy_id': engine.DEFAULT_THEORY}
(id1, _) = api['rule'].add_item(
{'rule': str(net_formula)}, {}, context=context)
# Poll
neutron = cage.service_object('neutron')
neutron2 = cage.service_object('neutron2')
neutron.poll()
neutron2.poll()
# Insert a second formula
other_formula = engine.parse1('q(x,y) :- p(x,y)')
(id2, _) = api['rule'].add_item(
{'rule': str(other_formula)}, {}, context=context)
ans1 = ('p("240ff9df-df35-43ae-9df5-27fae87f2492", '
' "240ff9df-df35-43ae-9df5-27fae87f2492") ')
ans2 = ('q("240ff9df-df35-43ae-9df5-27fae87f2492", '
' "240ff9df-df35-43ae-9df5-27fae87f2492") ')
# Wait for first query so messages can be delivered.
# But once the p table has its data, no need to wait anymore.
helper.retry_check_db_equal(engine, 'p(x,y)', ans1, target=policy)
e = helper.db_equal(engine.select('q(x,y)', target=policy), ans2)
self.assertTrue(e, "Insert rule-api 2")
# Get formula
ruleobj = api['rule'].get_item(id1, {}, context=context)
self.assertTrue(e, net_formula == engine.parse1(ruleobj['rule']))
# Get all formulas
ds = api['rule'].get_items({}, context=context)['results']
self.assertEqual(len(ds), 2)
ids = set([x['id'] for x in ds])
rules = set([engine.parse1(x['rule']) for x in ds])
self.assertEqual(ids, set([id1, id2]))
self.assertEqual(rules, set([net_formula, other_formula]))
# Delete formula
api['rule'].delete_item(id1, {}, context=context)
# Get all formulas
ds = api['rule'].get_items({}, context=context)['results']
self.assertEqual(len(ds), 1)
ids = sorted([x['id'] for x in ds])
self.assertEqual(ids, sorted([id2]))
def test_communication(self):
"""Test for communication.
Test the module's ability to be loaded into the DSE
by checking its ability to communicate on the message bus.
"""
cage = d6cage.d6Cage()
# Create modules.
# Turn off polling so we don't need to deal with real data.
args = helper.datasource_openstack_args()
args['poll_time'] = 0
cage.loadModule("NovaDriver",
helper.data_module_path("nova_driver.py"))
cage.loadModule("PolicyDriver", helper.policy_module_path())
cage.createservice(name="policy",
moduleName="PolicyDriver",
args={
'd6cage': cage,
'rootdir': helper.data_module_path('')
})
cage.createservice(name="nova", moduleName="NovaDriver", args=args)
# Check that data gets sent from nova to policy as expected
nova = cage.service_object('nova')
policy = cage.service_object('policy')
policy.debug_mode()
policy.create_policy('nova')
policy.set_schema('nova', compile.Schema({'server': (1, )}))
policy.subscribe('nova', 'server', callback=policy.receive_data)
# publishing is slightly convoluted b/c deltas are computed
# automatically. (Not just convenient--useful so that DSE
# properly handles the initial state problem.)
# Need to set nova.state and nova.prior_state and then publish
# anything.
# publish server(1), server(2), server(3)
helper.retry_check_subscribers(nova, [(policy.name, 'server')])
nova.prior_state = {}
nova.state['server'] = set([(1, ), (2, ), (3, )])
nova.publish('server', None)
helper.retry_check_db_equal(
policy, 'nova:server(x)',
'nova:server(1) nova:server(2) nova:server(3)')
# publish server(1), server(4), server(5)
nova.prior_state['server'] = nova.state['server']
nova.state['server'] = set([(1, ), (4, ), (5, )])
nova.publish('server', None)
helper.retry_check_db_equal(
policy, 'nova:server(x)',
'nova:server(1) nova:server(4) nova:server(5)')
def test_rule_api_model(self):
"""Test the rule api model.
Same as test_multiple except we use the api interface
instead of the DSE interface.
"""
api = self.api
cage = self.cage
engine = self.engine
policy = engine.DEFAULT_THEORY
# Insert formula
net_formula = test_neutron.create_networkXnetwork_group('p')
LOG.debug("Sending formula: %s", net_formula)
engine.debug_mode()
context = {'policy_id': engine.DEFAULT_THEORY}
(id1, rule) = api['rule'].add_item({'rule': str(net_formula)}, {},
context=context)
# Poll
neutron = cage.service_object('neutron')
neutron2 = cage.service_object('neutron2')
neutron.poll()
neutron2.poll()
# Insert a second formula
other_formula = engine.parse1('q(x,y) :- p(x,y)')
(id2, rule) = api['rule'].add_item({'rule': str(other_formula)}, {},
context=context)
ans1 = ('p("240ff9df-df35-43ae-9df5-27fae87f2492", '
' "240ff9df-df35-43ae-9df5-27fae87f2492") ')
ans2 = ('q("240ff9df-df35-43ae-9df5-27fae87f2492", '
' "240ff9df-df35-43ae-9df5-27fae87f2492") ')
# Wait for first query so messages can be delivered.
# But once the p table has its data, no need to wait anymore.
helper.retry_check_db_equal(engine, 'p(x,y)', ans1, target=policy)
e = helper.db_equal(engine.select('q(x,y)', target=policy), ans2)
self.assertTrue(e, "Insert rule-api 2")
# Get formula
ruleobj = api['rule'].get_item(id1, {}, context=context)
self.assertTrue(e, net_formula == engine.parse1(ruleobj['rule']))
# Get all formulas
ds = api['rule'].get_items({}, context=context)['results']
self.assertEqual(len(ds), 2)
ids = set([x['id'] for x in ds])
rules = set([engine.parse1(x['rule']) for x in ds])
self.assertEqual(ids, set([id1, id2]))
self.assertEqual(rules, set([net_formula, other_formula]))
# Delete formula
api['rule'].delete_item(id1, {}, context=context)
# Get all formulas
ds = api['rule'].get_items({}, context=context)['results']
self.assertEqual(len(ds), 1)
ids = sorted([x['id'] for x in ds])
self.assertEqual(ids, sorted([id2]))
def test_communication(self):
"""Test for communication.
Test the module's ability to be loaded into the DSE
by checking its ability to communicate on the message bus.
"""
cage = d6cage.d6Cage()
# Create modules.
# Turn off polling so we don't need to deal with real data.
args = helper.datasource_openstack_args()
args['poll_time'] = 0
cage.loadModule("NovaDriver",
helper.data_module_path("nova_driver.py"))
cage.loadModule("PolicyDriver", helper.policy_module_path())
cage.createservice(name="policy", moduleName="PolicyDriver",
args={'d6cage': cage,
'rootdir': helper.data_module_path(''),
'log_actions_only': True})
cage.createservice(name="nova", moduleName="NovaDriver", args=args)
# Check that data gets sent from nova to policy as expected
nova = cage.service_object('nova')
policy = cage.service_object('policy')
policy.debug_mode()
policy.create_policy('nova')
policy.set_schema('nova', compile.Schema({'server': (1,)}))
policy.subscribe('nova', 'server',
callback=policy.receive_data)
# publishing is slightly convoluted b/c deltas are computed
# automatically. (Not just convenient--useful so that DSE
# properly handles the initial state problem.)
# Need to set nova.state and nova.prior_state and then publish
# anything.
# publish server(1), server(2), server(3)
helper.retry_check_subscribers(nova, [(policy.name, 'server')])
nova.prior_state = {}
nova.state['server'] = set([(1,), (2,), (3,)])
nova.publish('server', None)
helper.retry_check_db_equal(
policy, 'nova:server(x)',
'nova:server(1) nova:server(2) nova:server(3)')
# publish server(1), server(4), server(5)
nova.prior_state['server'] = nova.state['server']
nova.state['server'] = set([(1,), (4,), (5,)])
nova.publish('server', None)
helper.retry_check_db_equal(
policy, 'nova:server(x)',
'nova:server(1) nova:server(4) nova:server(5)')
def test_double_poll_subscribe(self):
"""Test double polling before subscribing."""
cage = self.info['cage']
policy = cage.service_object('policy')
neutron = cage.service_object('neutron')
datalog2 = self.info['datalog2']
# poll twice and then subscribe: should see 2nd result
neutron.poll()
helper.retry_check_number_of_updates(neutron, 1)
neutron.poll()
helper.retry_check_number_of_updates(neutron, 2)
policy.subscribe('neutron', 'networks', callback=policy.receive_data)
helper.retry_check_db_equal(policy, 'p(x)', datalog2)
def test_double_poll_subscribe(self):
"""Test double polling before subscribing."""
cage = self.info["cage"]
policy = cage.service_object("policy")
neutron = cage.service_object("neutron")
datalog2 = self.info["datalog2"]
# poll twice and then subscribe: should see 2nd result
neutron.poll()
helper.retry_check_number_of_updates(neutron, 1)
neutron.poll()
helper.retry_check_number_of_updates(neutron, 2)
policy.subscribe("neutron", "networks", callback=policy.receive_data)
helper.retry_check_db_equal(policy, "p(x)", datalog2)
def test_double_poll_subscribe(self):
"""Test double polling before subscribing."""
cage = self.info['cage']
policy = cage.service_object('policy')
neutron = cage.service_object('neutron')
datalog2 = self.info['datalog2']
# poll twice and then subscribe: should see 2nd result
neutron.poll()
helper.retry_check_number_of_updates(neutron, 1)
neutron.poll()
helper.retry_check_number_of_updates(neutron, 2)
policy.subscribe('neutron', 'networks', callback=policy.receive_data)
helper.retry_check_db_equal(policy, 'p(x)', datalog2)
def test_policy(self):
node = helper.make_dsenode_new_partition('testnode')
data = FakeDataSource('data')
engine = Dse2Runtime('engine')
node.register_service(data)
node.register_service(engine)
engine.create_policy('alpha')
engine.create_policy('data')
self.insert_rule(engine, 'p(x) :- data:fake_table(x)', 'alpha')
data.state = {'fake_table': set([(1,), (2,)])}
data.poll()
helper.retry_check_db_equal(
engine, 'p(x)', 'p(1) p(2)', target='alpha')
self.assertFalse(hasattr(engine, "last_msg"))
def test_neutron(self):
"""Test polling and publishing of neutron updates."""
engine = self.engine
api = self.api
cage = self.cage
# Send formula
formula = test_neutron.create_network_group('p')
LOG.debug("Sending formula: %s", formula)
api['rule'].publish('policy-update', [runtime.Event(formula)])
helper.retry_check_nonempty_last_policy_change(engine)
LOG.debug("All services: %s", cage.services.keys())
neutron = cage.service_object('neutron')
neutron.poll()
ans = ('p("240ff9df-df35-43ae-9df5-27fae87f2492") ')
helper.retry_check_db_equal(engine, 'p(x)', ans)
def test_policy_tables(self):
"""Test basic DSE functionality with policy engine and the API."""
cage = congress.dse.d6cage.d6Cage()
cage.loadModule(
"TestDriver",
helper.data_module_path("../tests/datasources/test_driver.py"))
cage.loadModule("TestPolicy", helper.policy_module_path())
cage.createservice(name="data",
moduleName="TestDriver",
args=helper.datasource_openstack_args())
# using regular testdriver as API for now
cage.createservice(name="api",
moduleName="TestDriver",
args=helper.datasource_openstack_args())
cage.createservice(name="policy",
moduleName="TestPolicy",
args={
'd6cage': cage,
'rootdir': ''
})
data = cage.services['data']['object']
api = cage.services['api']['object']
policy = cage.services['policy']['object']
policy.create_policy('data')
policy.set_schema('data', compile.Schema({'q': (1, )}))
policy.subscribe('api',
'policy-update',
callback=policy.receive_policy_update)
# simulate API call for insertion of policy statements
formula = policy.parse1('p(x) :- data:q(x)')
api.publish('policy-update', [runtime.Event(formula)])
helper.retry_check_nonempty_last_policy_change(policy)
# simulate data source publishing to q
formula = policy.parse1('q(1)')
data.publish('q', [runtime.Event(formula)])
helper.retry_check_db_equal(policy, 'data:q(x)', 'data:q(1)')
# check that policy did the right thing with data
e = helper.db_equal(policy.select('p(x)'), 'p(1)')
self.assertTrue(e, 'Policy insert')
# check that publishing into 'p' does not work
formula = policy.parse1('p(3)')
data.publish('p', [runtime.Event(formula)])
# can't actually check that the update for p does not arrive
# so instead wait a bit and check
helper.pause()
e = helper.db_equal(policy.select('p(x)'), 'p(1)')
self.assertTrue(e, 'Policy non-insert')
def test_policy_data(self):
"""Test policy correctly processes initial data snapshot."""
node = helper.make_dsenode_new_partition('testnode')
node.always_snapshot = False
data = fake_datasource.FakeDataSource('data')
engine = agnostic.DseRuntime(api_base.ENGINE_SERVICE_ID)
node.register_service(data)
node.register_service(engine)
engine.create_policy('policy1')
engine.create_policy('data')
self.insert_rule(engine, 'p(x) :- data:fake_table(x)', 'policy1')
data.state = {'fake_table': set([(1,), (2,)])}
data.poll()
helper.retry_check_db_equal(
engine, 'p(x)', 'p(1) p(2)', target='policy1')
self.assertFalse(hasattr(engine, "last_msg"))
def test_policy_recovery(self):
"""Test policy crashing and recovering (sort of)."""
cage = self.info["cage"]
policy = cage.service_object("policy")
neutron = cage.service_object("neutron")
datalog1 = self.info["datalog1"]
# get initial data
policy.subscribe("neutron", "networks", callback=policy.receive_data)
helper.retry_check_subscribers(neutron, [(policy.name, "networks")])
neutron.poll()
helper.retry_check_db_equal(policy, "p(x)", datalog1)
# clear out policy's neutron:networks data (to simulate crashing)
policy.initialize_tables(["neutron:networks"], [])
# subscribe again (without unsubscribing)
policy.subscribe("neutron", "networks", callback=policy.receive_data)
helper.retry_check_db_equal(policy, "p(x)", datalog1)
def test_policy_recovery(self):
"""Test policy crashing and recovering (sort of)."""
cage = self.info['cage']
policy = cage.service_object('policy')
neutron = cage.service_object('neutron')
datalog1 = self.info['datalog1']
# get initial data
policy.subscribe('neutron', 'networks', callback=policy.receive_data)
helper.retry_check_subscribers(neutron, [(policy.name, 'networks')])
neutron.poll()
helper.retry_check_db_equal(policy, 'p(x)', datalog1)
# clear out policy's neutron:networks data (to simulate crashing)
policy.initialize_tables(['neutron:networks'], [])
# subscribe again (without unsubscribing)
policy.subscribe('neutron', 'networks', callback=policy.receive_data)
helper.retry_check_db_equal(policy, 'p(x)', datalog1)
def test_policy_recovery(self):
"""Test policy crashing and recovering (sort of)."""
cage = self.info['cage']
policy = cage.service_object('policy')
neutron = cage.service_object('neutron')
datalog1 = self.info['datalog1']
# get initial data
policy.subscribe('neutron', 'networks', callback=policy.receive_data)
helper.retry_check_subscribers(neutron, [(policy.name, 'networks')])
neutron.poll()
helper.retry_check_db_equal(policy, 'p(x)', datalog1)
# clear out policy's neutron:networks data (to simulate crashing)
policy.initialize_tables(['neutron:networks'], [])
# subscribe again (without unsubscribing)
policy.subscribe('neutron', 'networks', callback=policy.receive_data)
helper.retry_check_db_equal(policy, 'p(x)', datalog1)
def test_neutron(self):
"""Test polling and publishing of neutron updates."""
engine = self.engine
api = self.api
cage = self.cage
policy = engine.DEFAULT_THEORY
# Send formula
formula = test_neutron.create_network_group('p')
LOG.debug("Sending formula: %s", formula)
api['rule'].publish('policy-update',
[agnostic.Event(formula, target=policy)])
helper.retry_check_nonempty_last_policy_change(engine)
LOG.debug("All services: %s", cage.services.keys())
neutron = cage.service_object('neutron')
neutron.poll()
ans = ('p("240ff9df-df35-43ae-9df5-27fae87f2492") ')
helper.retry_check_db_equal(engine, 'p(x)', ans, target=policy)
def test_policy_initialization(self):
"""Test subscribing before polling. The common case."""
cage = self.info["cage"]
policy = cage.service_object("policy")
neutron = cage.service_object("neutron")
datalog1 = self.info["datalog1"]
fake_networks = self.info["fake_networks"]
# add garbage to policy
for formula in fake_networks:
policy.insert(formula)
# subscribe
policy.subscribe("neutron", "networks", callback=policy.receive_data)
helper.retry_check_subscribers(neutron, [(policy.name, "networks")])
# poll 1
neutron.poll()
helper.retry_check_db_equal(policy, "p(x)", datalog1)
def test_policy_initialization(self):
"""Test subscribing before polling. The common case."""
cage = self.info['cage']
policy = cage.service_object('policy')
neutron = cage.service_object('neutron')
datalog1 = self.info['datalog1']
fake_networks = self.info['fake_networks']
# add garbage to policy
for formula in fake_networks:
policy.insert(formula)
# subscribe
policy.subscribe('neutron', 'networks', callback=policy.receive_data)
helper.retry_check_subscribers(neutron, [(policy.name, 'networks')])
# poll 1
neutron.poll()
helper.retry_check_db_equal(policy, 'p(x)', datalog1)
def test_subscribe_poll(self):
"""Test subscribing before polling. The common case."""
cage = self.info['cage']
policy = cage.service_object('policy')
neutron = cage.service_object('neutron')
datalog1 = self.info['datalog1']
datalog2 = self.info['datalog2']
# subscribe
policy.subscribe('neutron', 'networks', callback=policy.receive_data)
helper.retry_check_subscribers(neutron, [(policy.name, 'networks')])
# poll 1
neutron.poll()
helper.retry_check_db_equal(policy, 'p(x)', datalog1)
# poll 2
neutron.poll()
helper.retry_check_db_equal(policy, 'p(x)', datalog2)
def test_subscribe_poll(self):
"""Test subscribing before polling. The common case."""
cage = self.info['cage']
policy = cage.service_object('policy')
neutron = cage.service_object('neutron')
datalog1 = self.info['datalog1']
datalog2 = self.info['datalog2']
# subscribe
policy.subscribe('neutron', 'networks', callback=policy.receive_data)
helper.retry_check_subscribers(neutron, [(policy.name, 'networks')])
# poll 1
neutron.poll()
helper.retry_check_db_equal(policy, 'p(x)', datalog1)
# poll 2
neutron.poll()
helper.retry_check_db_equal(policy, 'p(x)', datalog2)
def test_policy_initialization(self):
"""Test subscribing before polling. The common case."""
cage = self.info['cage']
policy = cage.service_object('policy')
neutron = cage.service_object('neutron')
datalog1 = self.info['datalog1']
fake_networks = self.info['fake_networks']
# add garbage to policy
for formula in fake_networks:
policy.insert(formula)
# subscribe
policy.subscribe('neutron', 'networks', callback=policy.receive_data)
helper.retry_check_subscribers(neutron, [(policy.name, 'networks')])
# poll 1
neutron.poll()
helper.retry_check_db_equal(policy, 'p(x)', datalog1)
def test_subscribe_poll(self):
"""Test subscribing before polling. The common case."""
cage = self.info["cage"]
policy = cage.service_object("policy")
neutron = cage.service_object("neutron")
datalog1 = self.info["datalog1"]
datalog2 = self.info["datalog2"]
# subscribe
policy.subscribe("neutron", "networks", callback=policy.receive_data)
helper.retry_check_subscribers(neutron, [(policy.name, "networks")])
# poll 1
neutron.poll()
helper.retry_check_db_equal(policy, "p(x)", datalog1)
# poll 2
neutron.poll()
helper.retry_check_db_equal(policy, "p(x)", datalog2)
def test_multiple(self):
"""Test polling and publishing of multiple neutron instances."""
api = self.api
cage = self.cage
engine = self.engine
# Send formula
formula = create_networkXnetwork_group('p')
api['rule'].publish('policy-update', [runtime.Event(formula)])
helper.retry_check_nonempty_last_policy_change(engine)
# poll datasources
neutron = cage.service_object('neutron')
neutron2 = cage.service_object('neutron2')
neutron.poll()
neutron2.poll()
# check answer
ans = ('p("240ff9df-df35-43ae-9df5-27fae87f2492", '
' "240ff9df-df35-43ae-9df5-27fae87f2492") ')
helper.retry_check_db_equal(engine, 'p(x,y)', ans)
def test_policy_data(self):
"""Test policy properly inserts data and processes it normally."""
cage = congress.dse.d6cage.d6Cage()
cage.loadModule("TestDriver",
helper.data_module_path("test_driver.py"))
cage.loadModule("TestPolicy", helper.policy_module_path())
cage.createservice(name="data", moduleName="TestDriver",
args=helper.datasource_openstack_args())
cage.createservice(name="policy", moduleName="TestPolicy",
args={'d6cage': cage, 'rootdir': ''})
data = cage.services['data']['object']
policy = cage.services['policy']['object']
# turn off module-schema syntax checking
policy.set_schema('data', compile.Schema({'p': (1,)}))
policy.subscribe('data', 'p', callback=policy.receive_data)
formula = compile.parse1('p(1)')
# sending a single Insert. (Default for Event is Insert.)
data.publish('p', [runtime.Event(formula)])
helper.retry_check_db_equal(policy, 'data:p(x)', 'data:p(1)')
def test_policy_tables(self):
"""Test basic DSE functionality with policy engine and the API."""
cage = congress.dse.d6cage.d6Cage()
cage.loadModule("TestDriver",
helper.data_module_path(
"../tests/datasources/test_driver.py"))
cage.loadModule("TestPolicy", helper.policy_module_path())
cage.createservice(name="data", moduleName="TestDriver",
args=helper.datasource_openstack_args())
# using regular testdriver as API for now
cage.createservice(name="api", moduleName="TestDriver",
args=helper.datasource_openstack_args())
cage.createservice(name="policy", moduleName="TestPolicy",
args={'d6cage': cage, 'rootdir': '',
'log_actions_only': True})
data = cage.services['data']['object']
api = cage.services['api']['object']
policy = cage.services['policy']['object']
policy.create_policy('data')
policy.set_schema('data', compile.Schema({'q': (1,)}))
policy.subscribe('api', 'policy-update',
callback=policy.receive_policy_update)
# simulate API call for insertion of policy statements
formula = policy.parse1('p(x) :- data:q(x)')
api.publish('policy-update', [compile.Event(formula)])
helper.retry_check_nonempty_last_policy_change(policy)
# simulate data source publishing to q
formula = policy.parse1('q(1)')
data.publish('q', [compile.Event(formula)])
helper.retry_check_db_equal(policy, 'data:q(x)', 'data:q(1)')
# check that policy did the right thing with data
e = helper.db_equal(policy.select('p(x)'), 'p(1)')
self.assertTrue(e, 'Policy insert')
# check that publishing into 'p' does not work
formula = policy.parse1('p(3)')
data.publish('p', [compile.Event(formula)])
# can't actually check that the update for p does not arrive
# so instead wait a bit and check
helper.pause()
e = helper.db_equal(policy.select('p(x)'), 'p(1)')
self.assertTrue(e, 'Policy non-insert')
def test_policy_data_late_sub(self):
"""Test policy correctly processes data on late subscribe."""
node = helper.make_dsenode_new_partition('testnode')
data = fake_datasource.FakeDataSource('data')
engine = agnostic.DseRuntime(api_base.ENGINE_SERVICE_ID)
node.register_service(data)
node.register_service(engine)
engine.create_policy('policy1')
engine.create_policy('data', kind=datalog_base.DATASOURCE_POLICY_TYPE)
data.state = {'fake_table': set([(1,), (2,)])}
data.poll()
self.insert_rule(engine, 'p(x) :- data:fake_table(x)', 'policy1')
helper.retry_check_db_equal(
engine, 'p(x)', 'p(1) p(2)', target='policy1')
data.state = {'fake_table': set([(1,), (2,), (3,)])}
data.poll()
helper.retry_check_db_equal(
engine, 'p(x)', 'p(1) p(2) p(3)', target='policy1')
self.assertFalse(hasattr(engine, "last_msg"))
node.stop()
def test_multiple(self):
"""Test polling and publishing of multiple neutron instances."""
api = self.api
cage = self.cage
engine = self.engine
policy = engine.DEFAULT_THEORY
# Send formula
formula = test_neutron.create_networkXnetwork_group('p')
api['rule'].publish('policy-update',
[agnostic.Event(formula, target=policy)])
helper.retry_check_nonempty_last_policy_change(engine)
# poll datasources
neutron = cage.service_object('neutron')
neutron2 = cage.service_object('neutron2')
neutron.poll()
neutron2.poll()
# check answer
ans = ('p("240ff9df-df35-43ae-9df5-27fae87f2492", '
' "240ff9df-df35-43ae-9df5-27fae87f2492") ')
helper.retry_check_db_equal(engine, 'p(x,y)', ans, target=policy)
def test_policy_data_late_sub(self):
"""Test policy correctly processes data on late subscribe."""
node = helper.make_dsenode_new_partition('testnode')
node.always_snapshot = False
data = FakeDataSource('data')
engine = Dse2Runtime('engine')
node.register_service(data)
node.register_service(engine)
engine.create_policy('policy1')
engine.create_policy('data')
data.state = {'fake_table': set([(1,), (2,)])}
data.poll()
self.insert_rule(engine, 'p(x) :- data:fake_table(x)', 'policy1')
helper.retry_check_db_equal(
engine, 'p(x)', 'p(1) p(2)', target='policy1')
data.state = {'fake_table': set([(1,), (2,), (3,)])}
data.poll()
helper.retry_check_db_equal(
engine, 'p(x)', 'p(1) p(2) p(3)', target='policy1')
self.assertFalse(hasattr(engine, "last_msg"))
def test_neutron_policy_poll_and_subscriptions(self):
"""Test polling and publishing of neutron updates."""
policy = self.engine.DEFAULT_THEORY
neutron2 = self._create_neutron_mock('neutron2')
self.engine.initialize_datasource('neutron',
self.neutronv2.get_schema())
self.engine.initialize_datasource('neutron2',
self.neutronv2.get_schema())
str_rule = ('p(x0, y0) :- neutron:networks(x0, x1, x2, x3, x4, x5), '
'neutron2:networks(y0, y1, y2, y3, y4, y5)')
rule = {'rule': str_rule, 'name': 'testrule1', 'comment': 'test'}
self.api['api-rule'].add_item(rule, {}, context={'policy_id': policy})
# Test policy subscriptions
subscriptions = self.engine.subscription_list()
self.assertEqual(sorted([('neutron', 'networks'),
('neutron2', 'networks')]), sorted(subscriptions))
# Test multiple instances
self.neutronv2.poll()
neutron2.poll()
ans = ('p("240ff9df-df35-43ae-9df5-27fae87f2492", '
' "240ff9df-df35-43ae-9df5-27fae87f2492") ')
helper.retry_check_db_equal(self.engine, 'p(x, y)', ans, target=policy)
def test_poll_subscribe(self):
"""Test polling before subscribing."""
cage = self.info['cage']
policy = cage.service_object('policy')
neutron = cage.service_object('neutron')
datalog1 = self.info['datalog1']
datalog2 = self.info['datalog2']
fake_networks = self.info['fake_networks']
# add garbage to policy
for formula in fake_networks:
policy.insert(formula)
# poll 1 and then subscribe; should still see first result
neutron.poll()
helper.retry_check_number_of_updates(neutron, 1)
policy.subscribe('neutron', 'networks', callback=policy.receive_data)
helper.retry_check_db_equal(policy, 'p(x)', datalog1)
# poll 2
neutron.poll()
helper.retry_check_db_equal(policy, 'p(x)', datalog2)
def test_poll_subscribe(self):
"""Test polling before subscribing."""
cage = self.info['cage']
policy = cage.service_object('policy')
neutron = cage.service_object('neutron')
datalog1 = self.info['datalog1']
datalog2 = self.info['datalog2']
fake_networks = self.info['fake_networks']
# add garbage to policy
for formula in fake_networks:
policy.insert(formula)
# poll 1 and then subscribe; should still see first result
neutron.poll()
helper.retry_check_number_of_updates(neutron, 1)
policy.subscribe('neutron', 'networks', callback=policy.receive_data)
helper.retry_check_db_equal(policy, 'p(x)', datalog1)
# poll 2
neutron.poll()
helper.retry_check_db_equal(policy, 'p(x)', datalog2)
def test_poll_subscribe(self):
"""Test polling before subscribing."""
cage = self.info["cage"]
policy = cage.service_object("policy")
neutron = cage.service_object("neutron")
datalog1 = self.info["datalog1"]
datalog2 = self.info["datalog2"]
fake_networks = self.info["fake_networks"]
# add garbage to policy
for formula in fake_networks:
policy.insert(formula)
# poll 1 and then subscribe; should still see first result
neutron.poll()
helper.retry_check_number_of_updates(neutron, 1)
policy.subscribe("neutron", "networks", callback=policy.receive_data)
helper.retry_check_db_equal(policy, "p(x)", datalog1)
# poll 2
neutron.poll()
helper.retry_check_db_equal(policy, "p(x)", datalog2)
def test_neutron_policy_poll_and_subscriptions(self):
"""Test polling and publishing of neutron updates."""
policy = self.engine.DEFAULT_THEORY
neutron2 = self._create_neutron_mock('neutron2')
self.engine.initialize_datasource('neutron',
self.neutronv2.get_schema())
self.engine.initialize_datasource('neutron2',
self.neutronv2.get_schema())
str_rule = ('p(x0, y0) :- neutron:networks(x0, x1, x2, x3, x4, x5), '
'neutron2:networks(y0, y1, y2, y3, y4, y5)')
rule = {'rule': str_rule, 'name': 'testrule1', 'comment': 'test'}
self.api['api-rule'].add_item(rule, {}, context={'policy_id': policy})
# Test policy subscriptions
subscriptions = self.engine.subscription_list()
self.assertEqual(sorted([('neutron', 'networks'),
('neutron2', 'networks')]), sorted(subscriptions))
# Test multiple instances
self.neutronv2.poll()
neutron2.poll()
ans = ('p("240ff9df-df35-43ae-9df5-27fae87f2492", '
' "240ff9df-df35-43ae-9df5-27fae87f2492") ')
helper.retry_check_db_equal(self.engine, 'p(x, y)', ans, target=policy)
def test_policy_table_publish(self):
"""Policy table result publish
Test basic DSE functionality with policy engine and table result
publish.
"""
node = helper.make_dsenode_new_partition('testnode')
data = fake_datasource.FakeDataSource('data')
policy = agnostic.DseRuntime('policy')
policy2 = agnostic.DseRuntime('policy2')
node.register_service(data)
node.register_service(policy)
node.register_service(policy2)
policy.synchronizer = mock.MagicMock()
policy2.synchronizer = mock.MagicMock()
policy.create_policy('data', kind=datalog_base.DATASOURCE_POLICY_TYPE)
policy.create_policy('classification')
policy.set_schema('data', compile.Schema({'q': (1, )}))
policy.insert('p(x):-data:q(x),gt(x,2)', target='classification')
policy.insert('q(3)', target='data')
# TODO(ekcs): test that no publish triggered (because no subscribers)
policy2.create_policy('policy')
policy2.subscribe('policy', 'classification:p')
helper.retry_check_function_return_value(
lambda: 'classification:p' in policy.
_published_tables_with_subscriber, True)
self.assertEqual(list(policy.policySubData.keys()),
[('p', 'classification', None)])
helper.retry_check_db_equal(policy2, 'policy:classification:p(x)',
'policy:classification:p(3)')
policy.insert('q(4)', target='data')
helper.retry_check_db_equal(policy2, 'policy:classification:p(x)',
('policy:classification:p(3)'
' policy:classification:p(4)'))
# test that no change to p means no publish triggered
policy.insert('q(2)', target='data')
# TODO(ekcs): test no publish triggered
policy.delete('q(4)', target='data')
helper.retry_check_db_equal(policy2, 'policy:classification:p(x)',
'policy:classification:p(3)')
policy2.unsubscribe('policy', 'classification:p')
# trigger removed
helper.retry_check_function_return_value(
lambda: len(policy._published_tables_with_subscriber) == 0, True)
self.assertEqual(list(policy.policySubData.keys()), [])
policy.insert('q(4)', target='data')
# TODO(ekcs): test that no publish triggered (because no subscribers)
node.stop()
def test_policy_table_publish(self):
"""Policy table result publish
Test basic DSE functionality with policy engine and table result
publish.
"""
node = helper.make_dsenode_new_partition('testnode')
data = fake_datasource.FakeDataSource('data')
policy = agnostic.DseRuntime('policy')
policy2 = agnostic.DseRuntime('policy2')
node.register_service(data)
node.register_service(policy)
node.register_service(policy2)
policy.create_policy('data')
policy.create_policy('classification')
policy.set_schema('data', compile.Schema({'q': (1,)}))
policy.insert('p(x):-data:q(x),gt(x,2)', target='classification')
policy.insert('q(3)', target='data')
# TODO(ekcs): test that no publish triggered (because no subscribers)
policy2.create_policy('policy')
policy2.subscribe('policy', 'classification:p')
helper.retry_check_function_return_value(
lambda: 'classification:p' in
policy._published_tables_with_subscriber, True)
self.assertEqual(list(policy.policySubData.keys()),
[('p', 'classification', None)])
helper.retry_check_db_equal(
policy2, 'policy:classification:p(x)',
'policy:classification:p(3)')
policy.insert('q(4)', target='data')
helper.retry_check_db_equal(
policy2, 'policy:classification:p(x)',
('policy:classification:p(3)'
' policy:classification:p(4)'))
# test that no change to p means no publish triggered
policy.insert('q(2)', target='data')
# TODO(ekcs): test no publish triggered
policy.delete('q(4)', target='data')
helper.retry_check_db_equal(
policy2, 'policy:classification:p(x)',
'policy:classification:p(3)')
policy2.unsubscribe('policy', 'classification:p')
# trigger removed
helper.retry_check_function_return_value(
lambda: len(policy._published_tables_with_subscriber) == 0, True)
self.assertEqual(list(policy.policySubData.keys()), [])
policy.insert('q(4)', target='data')