def test_receive_data_duplicate_sequence_number(self):
'''Test receiving data with duplicate sequence number
Only one message (arbitrary) should be processed.
'''
run = agnostic.Dse2Runtime('engine')
run.always_snapshot = False
run.create_policy('datasource1')
# send three updates with the same seqnum
run.receive_data_sequenced(
publisher='datasource1', table='p',
data=[[[1]], []], seqnum=1, is_snapshot=False)
run.receive_data_sequenced(
publisher='datasource1', table='p',
data=[[[2]], []], seqnum=1, is_snapshot=False)
run.receive_data_sequenced(
publisher='datasource1', table='p',
data=[[[3]], []], seqnum=1, is_snapshot=False)
# start with empty data
run.receive_data_sequenced(
publisher='datasource1', table='p',
data=[], seqnum=0, is_snapshot=True)
# exactly one of the three updates should be applied
actual = run.select('p(x)')
correct1 = 'p(1)'
correct2 = 'p(2)'
correct3 = 'p(3)'
self.assertTrue(
helper.db_equal(actual, correct1) or
helper.db_equal(actual, correct2) or
helper.db_equal(actual, correct3))
def test_poll_subscribe(self):
"""Test polling before subscribing."""
cage = self.info['cage']
policy = cage.service_object('policy')
neutron = cage.service_object('neutron')
datalog1 = self.info['datalog1']
datalog2 = self.info['datalog2']
fake_networks = self.info['fake_networks']
# add garbage to policy
for formula in fake_networks:
policy.insert(formula)
# poll 1 and then subscribe; should still see first result
neutron.poll()
helper.pause() # so that data is sent
policy.subscribe('neutron', 'networks', callback=policy.receive_data)
helper.pause() # so that data updates are processed
e = helper.db_equal(policy.select('p(x)'), datalog1)
self.assertTrue(e, 'Neutron insertion 1')
# poll 2
neutron.poll()
helper.pause()
e = helper.db_equal(policy.select('p(x)'), datalog2)
self.assertTrue(e, 'Neutron insertion 2')
def test_communication(self):
"""Test the module's ability to be loaded into the DSE
by checking its ability to communicate on the message bus.
"""
cage = congress.dse.d6cage.d6Cage()
# so that we exit once test finishes; all other threads are forced
# to be daemons
cage.daemon = True
cage.start()
# Create modules.
# Turn off polling so we don't need to deal with real data.
args = helper.datasource_openstack_args()
args['poll_time'] = 0
cage.loadModule("NovaDriver",
helper.data_module_path("nova_driver.py"))
cage.loadModule("PolicyDriver", helper.policy_module_path())
cage.createservice(name="policy", moduleName="PolicyDriver",
args={'d6cage': cage,
'rootdir': helper.data_module_path('')})
cage.createservice(name="nova", moduleName="NovaDriver", args=args)
# Check that data gets sent from nova to policy as expected
nova = cage.service_object('nova')
policy = cage.service_object('policy')
policy.subscribe('nova', 'server',
callback=policy.receive_data)
# publishing is slightly convoluted b/c deltas are computed
# automatically. (Not just convenient--useful so that DSE
# properly handles the initial state problem.)
# Need to set nova.state and nova.prior_state and then publish
# anything.
# publish server(1), server(2), server(3)
helper.pause()
nova.prior_state = {}
nova.state['server'] = set([(1,), (2,), (3,)])
nova.publish('server', None)
helper.pause()
e = helper.db_equal(
policy.select('nova:server(x)'),
'nova:server(1) nova:server(2) nova:server(3)')
self.assertTrue(e, 'Nova insertion 1')
# publish server(1), server(4), server(5)
helper.pause()
nova.prior_state['server'] = nova.state['server']
nova.state['server'] = set([(1,), (4,), (5,)])
nova.publish('server', None)
helper.pause()
e = helper.db_equal(
policy.select('nova:server(x)'),
'nova:server(1) nova:server(4) nova:server(5)')
self.assertTrue(e, 'Nova insertion 2')
def test_receive_data_no_sequence_num(self):
'''Test receiving data without sequence numbers'''
run = agnostic.DseRuntime(api_base.ENGINE_SERVICE_ID)
run.always_snapshot = False
run.create_policy('datasource1')
# initialize with full table
run.receive_data_sequenced(publisher='datasource1',
table='p',
data=[[1], [2]],
seqnum=None,
is_snapshot=True)
actual = run.select('p(x)')
correct = 'p(1) p(2)'
self.assertTrue(helper.db_equal(actual, correct))
# add data
run.receive_data_sequenced(publisher='datasource1',
table='p',
data=[[[3], [4]], []],
seqnum=None,
is_snapshot=False)
actual = run.select('p(x)')
correct = 'p(1) p(2) p(3) p(4)'
self.assertTrue(helper.db_equal(actual, correct))
# remove data
run.receive_data_sequenced(publisher='datasource1',
table='p',
data=[[], [[2], [4]]],
seqnum=None,
is_snapshot=False)
actual = run.select('p(x)')
correct = 'p(1) p(3)'
self.assertTrue(helper.db_equal(actual, correct))
# add & remove data
run.receive_data_sequenced(publisher='datasource1',
table='p',
data=[[[4]], [[3]]],
seqnum=None,
is_snapshot=False)
actual = run.select('p(x)')
correct = 'p(1) p(4)'
self.assertTrue(helper.db_equal(actual, correct))
# re-initialize with full table
run.receive_data_sequenced(publisher='datasource1',
table='p',
data=[[1], [2]],
seqnum=None,
is_snapshot=True)
actual = run.select('p(x)')
correct = 'p(1) p(2)'
self.assertTrue(helper.db_equal(actual, correct))
def test_receive_data_out_of_order(self):
'''Test receiving data with sequence numbers, out of order'''
run = agnostic.DseRuntime(api_base.ENGINE_SERVICE_ID)
run.always_snapshot = False
run.create_policy('datasource1')
# update with lower seqnum than init snapshot is ignored
run.receive_data_sequenced(publisher='datasource1',
table='p',
data=[[[10]], []],
seqnum=3,
is_snapshot=False)
# add & remove data
run.receive_data_sequenced(publisher='datasource1',
table='p',
data=[[[4]], [[3]]],
seqnum=7,
is_snapshot=False)
actual = run.select('p(x)')
correct = ''
self.assertTrue(helper.db_equal(actual, correct))
# remove data
run.receive_data_sequenced(publisher='datasource1',
table='p',
data=[[], [[2], [4]]],
seqnum=6,
is_snapshot=False)
actual = run.select('p(x)')
correct = ''
self.assertTrue(helper.db_equal(actual, correct))
# add data
run.receive_data_sequenced(publisher='datasource1',
table='p',
data=[[[3], [4]], []],
seqnum=5,
is_snapshot=False)
actual = run.select('p(x)')
correct = ''
self.assertTrue(helper.db_equal(actual, correct))
# initialize with full table
run.receive_data_sequenced(publisher='datasource1',
table='p',
data=[[1], [2]],
seqnum=4,
is_snapshot=True)
actual = run.select('p(x)')
correct = 'p(1) p(4)'
self.assertTrue(helper.db_equal(actual, correct))
def test_rule_api_model(self):
"""Test the rule api model. Same as test_multiple except
we use the api interface instead of the DSE interface.
"""
api = self.api
cage = self.cage
engine = self.engine
# Insert formula (which creates neutron services)
net_formula = create_networkXnetwork_group('p')
LOG.debug("Sending formula: {}".format(str(net_formula)))
engine.debug_mode()
context = {'policy_id': engine.DEFAULT_THEORY}
(id1, rule) = api['rule'].add_item(
{'rule': str(net_formula)}, {}, context=context)
# Poll
neutron = cage.service_object('neutron')
neutron2 = cage.service_object('neutron2')
neutron.poll()
neutron2.poll()
helper.pause()
# Insert a second formula
other_formula = compile.parse1('q(x,y) :- p(x,y)')
(id2, rule) = api['rule'].add_item(
{'rule': str(other_formula)}, {}, context=context)
helper.pause() # give time for messages/creation of services
ans1 = ('p("240ff9df-df35-43ae-9df5-27fae87f2492", '
' "240ff9df-df35-43ae-9df5-27fae87f2492") ')
ans2 = ('q("240ff9df-df35-43ae-9df5-27fae87f2492", '
' "240ff9df-df35-43ae-9df5-27fae87f2492") ')
e = helper.db_equal(engine.select('p(x,y)'), ans1)
self.assertTrue(e, "Insert rule-api 1")
e = helper.db_equal(engine.select('q(x,y)'), ans2)
self.assertTrue(e, "Insert rule-api 2")
# Get formula
ruleobj = api['rule'].get_item(id1, {}, context=context)
self.assertTrue(e, net_formula == compile.parse1(ruleobj['rule']))
# Get all formulas
ds = api['rule'].get_items({}, context=context)['results']
self.assertEqual(len(ds), 2)
ids = set([x['id'] for x in ds])
rules = set([compile.parse1(x['rule']) for x in ds])
self.assertEqual(ids, set([id1, id2]))
self.assertEqual(rules, set([net_formula, other_formula]))
# Delete formula
api['rule'].delete_item(id1, {}, context=context)
# Get all formulas
ds = api['rule'].get_items({}, context=context)['results']
self.assertEqual(len(ds), 1)
ids = sorted([x['id'] for x in ds])
self.assertEqual(ids, sorted([id2]))
def test_receive_data_sequence_number_max_int(self):
'''Test receiving data when sequence number goes over max int'''
run = agnostic.DseRuntime(api_base.ENGINE_SERVICE_ID)
run.always_snapshot = False
run.create_policy('datasource1')
run.receive_data_sequenced(publisher='datasource1',
table='p',
data=[[1], [2]],
seqnum=sys.maxsize,
is_snapshot=True)
actual = run.select('p(x)')
correct = 'p(1) p(2)'
self.assertTrue(helper.db_equal(actual, correct))
run.receive_data_sequenced(publisher='datasource1',
table='p',
data=[[], [[2]]],
seqnum=sys.maxsize + 1,
is_snapshot=False)
actual = run.select('p(x)')
correct = 'p(1)'
self.assertTrue(helper.db_equal(actual, correct))
# test out-of-sequence update ignored
run.receive_data_sequenced(publisher='datasource1',
table='p',
data=[[[2]], []],
seqnum=sys.maxsize,
is_snapshot=False)
actual = run.select('p(x)')
correct = 'p(1)'
self.assertTrue(helper.db_equal(actual, correct))
run.receive_data_sequenced(publisher='datasource1',
table='p',
data=[[[4]], []],
seqnum=sys.maxsize + 3,
is_snapshot=False)
actual = run.select('p(x)')
correct = 'p(1)'
self.assertTrue(helper.db_equal(actual, correct))
run.receive_data_sequenced(publisher='datasource1',
table='p',
data=[[[3]], []],
seqnum=sys.maxsize + 2,
is_snapshot=False)
actual = run.select('p(x)')
correct = 'p(1) p(3) p(4)'
self.assertTrue(helper.db_equal(actual, correct))
def test_policy_tables(self):
"""Test basic DSE functionality with policy engine and the API."""
cage = congress.dse.d6cage.d6Cage()
cage.loadModule(
"TestDriver",
helper.data_module_path("../tests/datasources/test_driver.py"))
cage.loadModule("TestPolicy", helper.policy_module_path())
cage.createservice(name="data",
moduleName="TestDriver",
args=helper.datasource_openstack_args())
# using regular testdriver as API for now
cage.createservice(name="api",
moduleName="TestDriver",
args=helper.datasource_openstack_args())
cage.createservice(name="policy",
moduleName="TestPolicy",
args={
'd6cage': cage,
'rootdir': ''
})
data = cage.services['data']['object']
api = cage.services['api']['object']
policy = cage.services['policy']['object']
policy.create_policy('data')
policy.set_schema('data', compile.Schema({'q': (1, )}))
policy.subscribe('api',
'policy-update',
callback=policy.receive_policy_update)
# simulate API call for insertion of policy statements
formula = policy.parse1('p(x) :- data:q(x)')
api.publish('policy-update', [runtime.Event(formula)])
helper.retry_check_nonempty_last_policy_change(policy)
# simulate data source publishing to q
formula = policy.parse1('q(1)')
data.publish('q', [runtime.Event(formula)])
helper.retry_check_db_equal(policy, 'data:q(x)', 'data:q(1)')
# check that policy did the right thing with data
e = helper.db_equal(policy.select('p(x)'), 'p(1)')
self.assertTrue(e, 'Policy insert')
# check that publishing into 'p' does not work
formula = policy.parse1('p(3)')
data.publish('p', [runtime.Event(formula)])
# can't actually check that the update for p does not arrive
# so instead wait a bit and check
helper.pause()
e = helper.db_equal(policy.select('p(x)'), 'p(1)')
self.assertTrue(e, 'Policy non-insert')
def test_receive_data_no_sequence_num(self):
'''Test receiving data without sequence numbers'''
run = agnostic.Dse2Runtime('engine')
run.always_snapshot = False
run.create_policy('datasource1')
# initialize with full table
run.receive_data_sequenced(
publisher='datasource1', table='p',
data=[[1], [2]], seqnum=None, is_snapshot=True)
actual = run.select('p(x)')
correct = 'p(1) p(2)'
self.assertTrue(helper.db_equal(actual, correct))
# add data
run.receive_data_sequenced(
publisher='datasource1', table='p',
data=[[[3], [4]], []], seqnum=None, is_snapshot=False)
actual = run.select('p(x)')
correct = 'p(1) p(2) p(3) p(4)'
self.assertTrue(helper.db_equal(actual, correct))
# remove data
run.receive_data_sequenced(
publisher='datasource1', table='p',
data=[[], [[2], [4]]], seqnum=None, is_snapshot=False)
actual = run.select('p(x)')
correct = 'p(1) p(3)'
self.assertTrue(helper.db_equal(actual, correct))
# add & remove data
run.receive_data_sequenced(
publisher='datasource1', table='p',
data=[[[4]], [[3]]], seqnum=None, is_snapshot=False)
actual = run.select('p(x)')
correct = 'p(1) p(4)'
self.assertTrue(helper.db_equal(actual, correct))
# re-initialize with full table
run.receive_data_sequenced(
publisher='datasource1', table='p',
data=[[1], [2]], seqnum=None, is_snapshot=True)
actual = run.select('p(x)')
correct = 'p(1) p(2)'
self.assertTrue(helper.db_equal(actual, correct))
def test_receive_data_in_order(self):
'''Test receiving data with sequence numbers, in order'''
run = agnostic.DseRuntime(api_base.ENGINE_SERVICE_ID)
run.create_policy('datasource1')
# initialize with full table
run.receive_data_sequenced(
publisher='datasource1', table='p',
data=[[1], [2]], seqnum=0, is_snapshot=True)
actual = run.select('p(x)')
correct = 'p(1) p(2)'
self.assertTrue(helper.db_equal(actual, correct))
# add data
run.receive_data_sequenced(
publisher='datasource1', table='p',
data=[[[3], [4]], []], seqnum=1, is_snapshot=False)
actual = run.select('p(x)')
correct = 'p(1) p(2) p(3) p(4)'
self.assertTrue(helper.db_equal(actual, correct))
# remove data
run.receive_data_sequenced(
publisher='datasource1', table='p',
data=[[], [[2], [4]]], seqnum=2, is_snapshot=False)
actual = run.select('p(x)')
correct = 'p(1) p(3)'
self.assertTrue(helper.db_equal(actual, correct))
# add & remove data
run.receive_data_sequenced(
publisher='datasource1', table='p',
data=[[[4]], [[3]]], seqnum=3, is_snapshot=False)
actual = run.select('p(x)')
correct = 'p(1) p(4)'
self.assertTrue(helper.db_equal(actual, correct))
# re-initialize with full table
run.receive_data_sequenced(
publisher='datasource1', table='p',
data=[[1], [2]], seqnum=4, is_snapshot=True)
actual = run.select('p(x)')
correct = 'p(1) p(2)'
self.assertTrue(helper.db_equal(actual, correct))
def check(self, run, action_sequence, query, correct, msg, delta=False):
original_db = str(run.theory[self.DEFAULT_THEORY])
actual = run.simulate(
query, self.DEFAULT_THEORY, action_sequence,
self.ACTION_THEORY, delta=delta)
e = helper.datalog_equal(actual, correct)
self.assertTrue(e, msg + " (Query results not correct)")
e = helper.db_equal(
str(run.theory[self.DEFAULT_THEORY]), original_db)
self.assertTrue(e, msg + " (Rollback failed)")
def test_receive_data_arbitrary_start(self):
'''Test receiving data with arbitrary starting sequence number'''
run = agnostic.DseRuntime(api_base.ENGINE_SERVICE_ID)
run.create_policy('datasource1')
run.receive_data_sequenced(
publisher='datasource1', table='p',
data=[[1], [2]], seqnum=1234, is_snapshot=True)
actual = run.select('p(x)')
correct = 'p(1) p(2)'
self.assertTrue(helper.db_equal(actual, correct))
def test_receive_data_out_of_order(self):
'''Test receiving data with sequence numbers, out of order'''
run = agnostic.Dse2Runtime('engine')
run.always_snapshot = False
run.create_policy('datasource1')
# update with lower seqnum than init snapshot is ignored
run.receive_data_sequenced(
publisher='datasource1', table='p',
data=[[[10]], []], seqnum=3, is_snapshot=False)
# add & remove data
run.receive_data_sequenced(
publisher='datasource1', table='p',
data=[[[4]], [[3]]], seqnum=7, is_snapshot=False)
actual = run.select('p(x)')
correct = ''
self.assertTrue(helper.db_equal(actual, correct))
# remove data
run.receive_data_sequenced(
publisher='datasource1', table='p',
data=[[], [[2], [4]]], seqnum=6, is_snapshot=False)
actual = run.select('p(x)')
correct = ''
self.assertTrue(helper.db_equal(actual, correct))
# add data
run.receive_data_sequenced(
publisher='datasource1', table='p',
data=[[[3], [4]], []], seqnum=5, is_snapshot=False)
actual = run.select('p(x)')
correct = ''
self.assertTrue(helper.db_equal(actual, correct))
# initialize with full table
run.receive_data_sequenced(
publisher='datasource1', table='p',
data=[[1], [2]], seqnum=4, is_snapshot=True)
actual = run.select('p(x)')
correct = 'p(1) p(4)'
self.assertTrue(helper.db_equal(actual, correct))
def test_receive_data_duplicate_sequence_number(self):
'''Test receiving data with duplicate sequence number
Only one message (arbitrary) should be processed.
'''
run = agnostic.DseRuntime(api_base.ENGINE_SERVICE_ID)
run.always_snapshot = False
run.create_policy('datasource1')
# send three updates with the same seqnum
run.receive_data_sequenced(publisher='datasource1',
table='p',
data=[[[1]], []],
seqnum=1,
is_snapshot=False)
run.receive_data_sequenced(publisher='datasource1',
table='p',
data=[[[2]], []],
seqnum=1,
is_snapshot=False)
run.receive_data_sequenced(publisher='datasource1',
table='p',
data=[[[3]], []],
seqnum=1,
is_snapshot=False)
# start with empty data
run.receive_data_sequenced(publisher='datasource1',
table='p',
data=[],
seqnum=0,
is_snapshot=True)
# exactly one of the three updates should be applied
actual = run.select('p(x)')
correct1 = 'p(1)'
correct2 = 'p(2)'
correct3 = 'p(3)'
self.assertTrue(
helper.db_equal(actual, correct1)
or helper.db_equal(actual, correct2)
or helper.db_equal(actual, correct3))
def test_policy_creation_after_ref(self):
"""Test ability to write rules that span multiple policies."""
# Local table used
run = agnostic.Runtime()
run.create_policy('test1')
run.insert('p(x) :- test2:q(x)', 'test1')
run.create_policy('test2')
run.insert('q(1)', 'test2')
actual = run.select('p(x)', 'test1')
e = helper.db_equal(actual, 'p(1)')
self.assertTrue(e, "Creation after reference")
def check(self, run, action_sequence, query, correct, msg, delta=False):
original_db = str(run.theory[self.DEFAULT_THEORY])
actual = run.simulate(query,
self.DEFAULT_THEORY,
action_sequence,
self.ACTION_THEORY,
delta=delta)
e = helper.datalog_equal(actual, correct)
self.assertTrue(e, msg + " (Query results not correct)")
e = helper.db_equal(str(run.theory[self.DEFAULT_THEORY]), original_db)
self.assertTrue(e, msg + " (Rollback failed)")
def test_policy_creation_after_ref(self):
"""Test ability to write rules that span multiple policies."""
# Local table used
run = agnostic.Runtime()
run.create_policy('test1')
run.insert('p(x) :- test2:q(x)', 'test1')
run.create_policy('test2')
run.insert('q(1)', 'test2')
actual = run.select('p(x)', 'test1')
e = helper.db_equal(actual, 'p(1)')
self.assertTrue(e, "Creation after reference")
def test_policy_tables(self):
"""Test basic DSE functionality with policy engine and the API."""
cage = congress.dse.d6cage.d6Cage()
cage.loadModule("TestDriver",
helper.data_module_path(
"../tests/datasources/test_driver.py"))
cage.loadModule("TestPolicy", helper.policy_module_path())
cage.createservice(name="data", moduleName="TestDriver",
args=helper.datasource_openstack_args())
# using regular testdriver as API for now
cage.createservice(name="api", moduleName="TestDriver",
args=helper.datasource_openstack_args())
cage.createservice(name="policy", moduleName="TestPolicy",
args={'d6cage': cage, 'rootdir': '',
'log_actions_only': True})
data = cage.services['data']['object']
api = cage.services['api']['object']
policy = cage.services['policy']['object']
policy.create_policy('data')
policy.set_schema('data', compile.Schema({'q': (1,)}))
policy.subscribe('api', 'policy-update',
callback=policy.receive_policy_update)
# simulate API call for insertion of policy statements
formula = policy.parse1('p(x) :- data:q(x)')
api.publish('policy-update', [compile.Event(formula)])
helper.retry_check_nonempty_last_policy_change(policy)
# simulate data source publishing to q
formula = policy.parse1('q(1)')
data.publish('q', [compile.Event(formula)])
helper.retry_check_db_equal(policy, 'data:q(x)', 'data:q(1)')
# check that policy did the right thing with data
e = helper.db_equal(policy.select('p(x)'), 'p(1)')
self.assertTrue(e, 'Policy insert')
# check that publishing into 'p' does not work
formula = policy.parse1('p(3)')
data.publish('p', [compile.Event(formula)])
# can't actually check that the update for p does not arrive
# so instead wait a bit and check
helper.pause()
e = helper.db_equal(policy.select('p(x)'), 'p(1)')
self.assertTrue(e, 'Policy non-insert')
def test_policy_tables(self):
"""Test basic DSE functionality with policy engine and the API."""
cage = congress.dse.d6cage.d6Cage()
# so that we exit once test finishes; all other threads are forced
# to be daemons
cage.daemon = True
cage.start()
cage.loadModule("TestDriver",
helper.data_module_path("test_driver.py"))
cage.loadModule("TestPolicy", helper.policy_module_path())
cage.createservice(name="data", moduleName="TestDriver",
args=helper.datasource_openstack_args())
# using regular testdriver as API for now
cage.createservice(name="api", moduleName="TestDriver",
args=helper.datasource_openstack_args())
cage.createservice(name="policy", moduleName="TestPolicy",
args={'d6cage': cage, 'rootdir': ''})
data = cage.services['data']['object']
api = cage.services['api']['object']
policy = cage.services['policy']['object']
policy.subscribe('api', 'policy-update',
callback=policy.receive_policy_update)
# simulate API call for insertion of policy statements
formula = compile.parse1('p(x) :- data:q(x)')
api.publish('policy-update', [runtime.Event(formula)])
helper.pause()
# simulate data source publishing to q
formula = compile.parse1('q(1)')
data.publish('q', [runtime.Event(formula)])
helper.pause() # give other threads chance to run
# check that policy did the right thing with data
e = helper.db_equal(policy.select('data:q(x)'), 'data:q(1)')
self.assertTrue(e, 'Policy insert 1')
e = helper.db_equal(policy.select('p(x)'), 'p(1)')
self.assertTrue(e, 'Policy insert 2')
#check that publishing into 'p' does not work
formula = compile.parse1('p(3)')
data.publish('p', [runtime.Event(formula)])
helper.pause()
e = helper.db_equal(policy.select('p(x)'), 'p(1)')
self.assertTrue(e, 'Policy noninsert')
def test_receive_data_sequence_number_max_int(self):
'''Test receiving data when sequence number goes over max int'''
run = agnostic.Dse2Runtime('engine')
run.always_snapshot = False
run.create_policy('datasource1')
run.receive_data_sequenced(
publisher='datasource1', table='p',
data=[[1], [2]], seqnum=sys.maxsize, is_snapshot=True)
actual = run.select('p(x)')
correct = 'p(1) p(2)'
self.assertTrue(helper.db_equal(actual, correct))
run.receive_data_sequenced(
publisher='datasource1', table='p',
data=[[], [[2]]], seqnum=sys.maxsize + 1, is_snapshot=False)
actual = run.select('p(x)')
correct = 'p(1)'
self.assertTrue(helper.db_equal(actual, correct))
# test out-of-sequence update ignored
run.receive_data_sequenced(
publisher='datasource1', table='p',
data=[[[2]], []], seqnum=sys.maxsize, is_snapshot=False)
actual = run.select('p(x)')
correct = 'p(1)'
self.assertTrue(helper.db_equal(actual, correct))
run.receive_data_sequenced(
publisher='datasource1', table='p',
data=[[[4]], []], seqnum=sys.maxsize + 3, is_snapshot=False)
actual = run.select('p(x)')
correct = 'p(1)'
self.assertTrue(helper.db_equal(actual, correct))
run.receive_data_sequenced(
publisher='datasource1', table='p',
data=[[[3]], []], seqnum=sys.maxsize + 2, is_snapshot=False)
actual = run.select('p(x)')
correct = 'p(1) p(3) p(4)'
self.assertTrue(helper.db_equal(actual, correct))
def test_external(self):
"""Test ability to write rules that span multiple policies."""
# External theory
run = runtime.Runtime()
run.create_policy('test1')
run.insert('q(1)', target='test1')
run.insert('q(2)', target='test1')
run.create_policy('test2')
run.insert('p(x) :- test1:q(x)', target='test2')
actual = run.select('p(x)', target='test2')
e = helper.db_equal(actual, 'p(1) p(2)')
self.assertTrue(e, "Basic")
def test_receive_data_arbitrary_start(self):
'''Test receiving data with arbitrary starting sequence number'''
run = agnostic.DseRuntime(api_base.ENGINE_SERVICE_ID)
run.create_policy('datasource1')
run.receive_data_sequenced(publisher='datasource1',
table='p',
data=[[1], [2]],
seqnum=1234,
is_snapshot=True)
actual = run.select('p(x)')
correct = 'p(1) p(2)'
self.assertTrue(helper.db_equal(actual, correct))
def test_rule_api_model(self):
"""Test the rule api model.
Same as test_multiple except we use the api interface
instead of the DSE interface.
"""
api = self.api
cage = self.cage
engine = self.engine
policy = engine.DEFAULT_THEORY
# Insert formula
net_formula = test_neutron.create_networkXnetwork_group('p')
LOG.debug("Sending formula: %s", net_formula)
engine.debug_mode()
context = {'policy_id': engine.DEFAULT_THEORY}
(id1, _) = api['rule'].add_item(
{'rule': str(net_formula)}, {}, context=context)
# Poll
neutron = cage.service_object('neutron')
neutron2 = cage.service_object('neutron2')
neutron.poll()
neutron2.poll()
# Insert a second formula
other_formula = engine.parse1('q(x,y) :- p(x,y)')
(id2, _) = api['rule'].add_item(
{'rule': str(other_formula)}, {}, context=context)
ans1 = ('p("240ff9df-df35-43ae-9df5-27fae87f2492", '
' "240ff9df-df35-43ae-9df5-27fae87f2492") ')
ans2 = ('q("240ff9df-df35-43ae-9df5-27fae87f2492", '
' "240ff9df-df35-43ae-9df5-27fae87f2492") ')
# Wait for first query so messages can be delivered.
# But once the p table has its data, no need to wait anymore.
helper.retry_check_db_equal(engine, 'p(x,y)', ans1, target=policy)
e = helper.db_equal(engine.select('q(x,y)', target=policy), ans2)
self.assertTrue(e, "Insert rule-api 2")
# Get formula
ruleobj = api['rule'].get_item(id1, {}, context=context)
self.assertTrue(e, net_formula == engine.parse1(ruleobj['rule']))
# Get all formulas
ds = api['rule'].get_items({}, context=context)['results']
self.assertEqual(len(ds), 2)
ids = set([x['id'] for x in ds])
rules = set([engine.parse1(x['rule']) for x in ds])
self.assertEqual(ids, set([id1, id2]))
self.assertEqual(rules, set([net_formula, other_formula]))
# Delete formula
api['rule'].delete_item(id1, {}, context=context)
# Get all formulas
ds = api['rule'].get_items({}, context=context)['results']
self.assertEqual(len(ds), 1)
ids = sorted([x['id'] for x in ds])
self.assertEqual(ids, sorted([id2]))
def test_rule_api_model(self):
"""Test the rule api model.
Same as test_multiple except we use the api interface
instead of the DSE interface.
"""
api = self.api
cage = self.cage
engine = self.engine
policy = engine.DEFAULT_THEORY
# Insert formula
net_formula = test_neutron.create_networkXnetwork_group('p')
LOG.debug("Sending formula: %s", net_formula)
engine.debug_mode()
context = {'policy_id': engine.DEFAULT_THEORY}
(id1, rule) = api['rule'].add_item({'rule': str(net_formula)}, {},
context=context)
# Poll
neutron = cage.service_object('neutron')
neutron2 = cage.service_object('neutron2')
neutron.poll()
neutron2.poll()
# Insert a second formula
other_formula = engine.parse1('q(x,y) :- p(x,y)')
(id2, rule) = api['rule'].add_item({'rule': str(other_formula)}, {},
context=context)
ans1 = ('p("240ff9df-df35-43ae-9df5-27fae87f2492", '
' "240ff9df-df35-43ae-9df5-27fae87f2492") ')
ans2 = ('q("240ff9df-df35-43ae-9df5-27fae87f2492", '
' "240ff9df-df35-43ae-9df5-27fae87f2492") ')
# Wait for first query so messages can be delivered.
# But once the p table has its data, no need to wait anymore.
helper.retry_check_db_equal(engine, 'p(x,y)', ans1, target=policy)
e = helper.db_equal(engine.select('q(x,y)', target=policy), ans2)
self.assertTrue(e, "Insert rule-api 2")
# Get formula
ruleobj = api['rule'].get_item(id1, {}, context=context)
self.assertTrue(e, net_formula == engine.parse1(ruleobj['rule']))
# Get all formulas
ds = api['rule'].get_items({}, context=context)['results']
self.assertEqual(len(ds), 2)
ids = set([x['id'] for x in ds])
rules = set([engine.parse1(x['rule']) for x in ds])
self.assertEqual(ids, set([id1, id2]))
self.assertEqual(rules, set([net_formula, other_formula]))
# Delete formula
api['rule'].delete_item(id1, {}, context=context)
# Get all formulas
ds = api['rule'].get_items({}, context=context)['results']
self.assertEqual(len(ds), 1)
ids = sorted([x['id'] for x in ds])
self.assertEqual(ids, sorted([id2]))
def test_external(self):
"""Test ability to write rules that span multiple policies."""
# External theory
run = agnostic.Runtime()
run.create_policy('test1')
run.insert('q(1)', target='test1')
run.insert('q(2)', target='test1')
run.create_policy('test2')
run.insert('p(x) :- test1:q(x)', target='test2')
actual = run.select('p(x)', target='test2')
e = helper.db_equal(actual, 'p(1) p(2)')
self.assertTrue(e, "Basic")
def test_multipolicy_head(self):
"""Test SELECT with different policy in the head."""
run = agnostic.Runtime()
run.debug_mode()
run.create_policy('test1', kind='action')
run.create_policy('test2', kind='action')
(permitted, errors) = run.insert('test2:p+(x) :- q(x)', 'test1')
self.assertTrue(permitted, "modals with policy names must be allowed")
run.insert('q(1)', 'test1')
run.insert('p(2)', 'test2')
actual = run.select('test2:p+(x)', 'test1')
e = helper.db_equal(actual, 'test2:p+(1)')
self.assertTrue(e, "Policy name in the head")
def test_local(self):
"""Test ability to write rules that span multiple policies."""
# Local table used
run = runtime.Runtime()
run.create_policy('test1')
run.insert('q(1)', target='test1')
run.insert('q(2)', target='test1')
run.create_policy('test2')
run.insert('p(x) :- test1:q(x), q(x)', target='test2')
run.insert('q(2)', 'test2')
actual = run.select('p(x)', target='test2')
e = helper.db_equal(actual, 'p(2)')
self.assertTrue(e, "Local table used")
def test_local(self):
"""Test ability to write rules that span multiple policies."""
# Local table used
run = agnostic.Runtime()
run.create_policy('test1')
run.insert('q(1)', target='test1')
run.insert('q(2)', target='test1')
run.create_policy('test2')
run.insert('p(x) :- test1:q(x), q(x)', target='test2')
run.insert('q(2)', 'test2')
actual = run.select('p(x)', target='test2')
e = helper.db_equal(actual, 'p(2)')
self.assertTrue(e, "Local table used")
def test_multipolicy_head(self):
"""Test SELECT with different policy in the head."""
run = runtime.Runtime()
run.debug_mode()
run.create_policy('test1', kind='action')
run.create_policy('test2', kind='action')
(permitted, errors) = run.insert('test2:p+(x) :- q(x)', 'test1')
self.assertTrue(permitted, "modals with policy names must be allowed")
run.insert('q(1)', 'test1')
run.insert('p(2)', 'test2')
actual = run.select('test2:p+(x)', 'test1')
e = helper.db_equal(actual, 'test2:p+(1)')
self.assertTrue(e, "Policy name in the head")
def test_multi_external(self):
"""Test multiple rules that span multiple policies."""
run = agnostic.Runtime()
run.debug_mode()
run.create_policy('test1')
run.create_policy('test2')
run.create_policy('test3')
run.insert('p(x) :- test2:p(x)', target='test1')
run.insert('p(x) :- test3:p(x)', target='test1')
run.insert('p(1)', target='test2')
run.insert('p(2)', target='test3')
actual = run.select('p(x)', target='test1')
e = helper.db_equal(actual, 'p(1) p(2)')
self.assertTrue(e, "Multiple external rules with multiple policies")
def test_subscribe_poll(self):
"""Test subscribing before polling. The common case."""
cage = self.info['cage']
policy = cage.service_object('policy')
neutron = cage.service_object('neutron')
datalog1 = self.info['datalog1']
datalog2 = self.info['datalog2']
# subscribe
policy.subscribe('neutron', 'networks', callback=policy.receive_data)
helper.pause() # so that subscription messages are processed
# poll 1
neutron.poll()
helper.pause(3) # so that data updates are processed
e = helper.db_equal(policy.select('p(x)'), datalog1)
self.assertTrue(e, 'Neutron insertion 1: ' + str(policy.content()))
# poll 2
neutron.poll()
helper.pause()
e = helper.db_equal(policy.select('p(x)'), datalog2)
self.assertTrue(e, 'Neutron insertion 2')
def test_policy_recovery(self):
"""Test policy crashing and recovering (sort of)."""
cage = self.info['cage']
policy = cage.service_object('policy')
neutron = cage.service_object('neutron')
datalog1 = self.info['datalog1']
# get initial data
policy.subscribe('neutron', 'networks', callback=policy.receive_data)
helper.pause()
neutron.poll()
helper.pause()
e = helper.db_equal(policy.select('p(x)'), datalog1)
self.assertTrue(e, 'Neutron insertion 1')
# clear out policy's neutron:networks data (to simulate crashing)
policy.initialize(['neutron:networks'], [])
# subscribe again (without unsubscribing)
policy.subscribe('neutron', 'networks', callback=policy.receive_data)
helper.pause()
# should get same data
e = helper.db_equal(policy.select('p(x)'), datalog1)
self.assertTrue(e, 'Neutron insertion 1')
def test_external_current(self):
"""Test ability to write rules that span multiple policies."""
# External theory plus current theory
run = agnostic.Runtime()
run.create_policy('test1')
run.insert('q(1)', target='test1')
run.insert('q(2)', target='test1')
run.create_policy('test2')
run.insert('p(x) :- test1:q(x), r(x)', target='test2')
run.insert('r(1)', target='test2')
run.insert('r(2)', target='test2')
actual = run.select('p(x)', target='test2')
e = helper.db_equal(actual, 'p(1) p(2)')
self.assertTrue(e, "Mixing external theories with current theory")
def test_multi_external(self):
"""Test multiple rules that span multiple policies."""
run = runtime.Runtime()
run.debug_mode()
run.create_policy('test1')
run.create_policy('test2')
run.create_policy('test3')
run.insert('p(x) :- test2:p(x)', target='test1')
run.insert('p(x) :- test3:p(x)', target='test1')
run.insert('p(1)', target='test2')
run.insert('p(2)', target='test3')
actual = run.select('p(x)', target='test1')
e = helper.db_equal(actual, 'p(1) p(2)')
self.assertTrue(e, "Multiple external rules with multiple policies")
def test_double_poll_subscribe(self):
"""Test double polling before subscribing."""
cage = self.info['cage']
policy = cage.service_object('policy')
neutron = cage.service_object('neutron')
datalog2 = self.info['datalog2']
# poll twice and then subscribe: should see 2nd result
neutron.poll()
helper.pause()
neutron.poll()
helper.pause()
policy.subscribe('neutron', 'networks', callback=policy.receive_data)
helper.pause() # so that messages are processed
e = helper.db_equal(policy.select('p(x)'), datalog2)
self.assertTrue(e, 'Neutron insertion 2')
def test_multiple_external(self):
"""Test ability to write rules that span multiple policies."""
# Multiple external theories
run = agnostic.Runtime()
run.create_policy('test1')
run.insert('q(1)', target='test1')
run.insert('q(2)', target='test1')
run.insert('q(3)', target='test1')
run.create_policy('test2')
run.insert('q(1)', target='test2')
run.insert('q(2)', target='test2')
run.insert('q(4)', target='test2')
run.create_policy('test3')
run.insert('p(x) :- test1:q(x), test2:q(x)', target='test3')
actual = run.select('p(x)', target='test3')
e = helper.db_equal(actual, 'p(1) p(2)')
self.assertTrue(e, "Multiple external theories")
def test_receive_data_multiple_tables(self):
'''Test receiving data with sequence numbers, multiple tables'''
run = agnostic.Dse2Runtime('engine')
run.always_snapshot = False
run.create_policy('datasource1')
# initialize p with full table
run.receive_data_sequenced(
publisher='datasource1', table='p',
data=[[1]], seqnum=0, is_snapshot=True)
actual = run.select('p(x)')
correct = 'p(1)'
self.assertTrue(helper.db_equal(actual, correct))
# add data to p
run.receive_data_sequenced(
publisher='datasource1', table='p',
data=[[[2]], []], seqnum=1, is_snapshot=False)
actual = run.select('p(x)')
correct = 'p(1) p(2)'
self.assertTrue(helper.db_equal(actual, correct))
# add data to q
run.receive_data_sequenced(
publisher='datasource1', table='q',
data=[[[2]], []], seqnum=1, is_snapshot=False)
actual = run.select('q(x)')
correct = '' # does not apply until initialize
self.assertTrue(helper.db_equal(actual, correct))
# initialize q with full table
run.receive_data_sequenced(
publisher='datasource1', table='q',
data=[[1]], seqnum=0, is_snapshot=True)
actual = run.select('q(x)')
correct = 'q(1) q(2)' # both initial data and preceding update applied
self.assertTrue(helper.db_equal(actual, correct))
# add data to q
run.receive_data_sequenced(
publisher='datasource1', table='q',
data=[[[3]], []], seqnum=2, is_snapshot=False)
actual = run.select('q(x)')
correct = 'q(1) q(2) q(3)'
self.assertTrue(helper.db_equal(actual, correct))
# add data to p
run.receive_data_sequenced(
publisher='datasource1', table='p',
data=[[[3]], []], seqnum=2, is_snapshot=False)
actual = run.select('p(x)')
correct = 'p(1) p(2) p(3)'
self.assertTrue(helper.db_equal(actual, correct))
def test_neutron(self):
"""Test polling and publishing of neutron updates."""
engine = self.engine
api = self.api
cage = self.cage
helper.pause()
# Send formula
formula = test_neutron.create_network_group('p')
LOG.debug("Sending formula: {}".format(str(formula)))
api['rule'].publish('policy-update', [runtime.Event(formula)])
helper.pause() # give time for messages/creation of services
LOG.debug("All services: " + str(cage.services.keys()))
neutron = cage.service_object('neutron')
neutron.poll()
helper.pause()
ans = ('p("240ff9df-df35-43ae-9df5-27fae87f2492") ')
e = helper.db_equal(engine.select('p(x)'), ans)
self.assertTrue(e, "Neutron datasource")
def test_multiple_levels_external(self):
"""Test ability to write rules that span multiple policies."""
# Multiple levels of external theories
run = runtime.Runtime()
run.debug_mode()
run.create_policy('test1')
run.insert('p(x) :- test2:q(x), test3:q(x)', target='test1')
run.insert('s(3) s(1) s(2) s(4)', target='test1')
run.create_policy('test2')
run.insert('q(x) :- test4:r(x)', target='test2')
run.create_policy('test3')
run.insert('q(x) :- test1:s(x)', target='test3')
run.create_policy('test4')
run.insert('r(1)', target='test4')
run.insert('r(2)', target='test4')
run.insert('r(5)', target='test4')
actual = run.select('p(x)', target='test1')
e = helper.db_equal(actual, 'p(1) p(2)')
self.assertTrue(e, "Multiple levels of external theories")
def test_multiple_levels_external(self):
"""Test ability to write rules that span multiple policies."""
# Multiple levels of external theories
run = agnostic.Runtime()
run.debug_mode()
run.create_policy('test1')
run.insert('p(x) :- test2:q(x), test3:q(x)', target='test1')
run.insert('s(3) s(1) s(2) s(4)', target='test1')
run.create_policy('test2')
run.insert('q(x) :- test4:r(x)', target='test2')
run.create_policy('test3')
run.insert('q(x) :- test1:s(x)', target='test3')
run.create_policy('test4')
run.insert('r(1)', target='test4')
run.insert('r(2)', target='test4')
run.insert('r(5)', target='test4')
actual = run.select('p(x)', target='test1')
e = helper.db_equal(actual, 'p(1) p(2)')
self.assertTrue(e, "Multiple levels of external theories")
def test_multiple(self):
"""Test polling and publishing of multiple neutron instances."""
api = self.api
cage = self.cage
engine = self.engine
# Send formula
formula = create_networkXnetwork_group('p')
api['rule'].publish('policy-update', [runtime.Event(formula)])
helper.pause() # give time for messages/creation of services
# poll datasources
neutron = cage.service_object('neutron')
neutron2 = cage.service_object('neutron2')
neutron.poll()
neutron2.poll()
helper.pause()
# check answer
ans = ('p("240ff9df-df35-43ae-9df5-27fae87f2492", '
' "240ff9df-df35-43ae-9df5-27fae87f2492") ')
e = helper.db_equal(engine.select('p(x,y)'), ans)
self.assertTrue(e, "Multiple neutron datasources")
def test_policy_initialization(self):
"""Test subscribing before polling. The common case."""
cage = self.info['cage']
policy = cage.service_object('policy')
neutron = cage.service_object('neutron')
datalog1 = self.info['datalog1']
fake_networks = self.info['fake_networks']
# add garbage to policy
for formula in fake_networks:
policy.insert(formula)
# subscribe
policy.subscribe('neutron', 'networks', callback=policy.receive_data)
helper.pause() # so that subscription messages are processed
# poll 1
neutron.poll()
helper.pause() # so that data updates are processed
e = helper.db_equal(policy.select('p(x)'), datalog1)
self.assertTrue(e, 'Neutron insertion 1')
def test_policy_data(self):
"""Test policy properly inserts data and processes it normally."""
cage = congress.dse.d6cage.d6Cage()
# so that we exit once test finishes; all other threads are forced
# to be daemons
cage.daemon = True
cage.start()
cage.loadModule("TestDriver",
helper.data_module_path("test_driver.py"))
cage.loadModule("TestPolicy", helper.policy_module_path())
cage.createservice(name="data", moduleName="TestDriver")
cage.createservice(name="policy", moduleName="TestPolicy")
data = cage.services['data']['object']
policy = cage.services['policy']['object']
policy.subscribe('data', 'p', callback=policy.receive_data)
formula = compile.parse1('p(1)')
# sending a single Insert. (Default for Event is Insert.)
data.publish('p', [runtime.Event(formula)])
helper.pause() # give other threads chance to run
e = helper.db_equal(policy.select('data:p(x)'), 'data:p(1)')
self.assertTrue(e, 'Single insert')
def test_receive_data_multiple_tables(self):
'''Test receiving data with sequence numbers, multiple tables'''
run = agnostic.DseRuntime(api_base.ENGINE_SERVICE_ID)
run.create_policy('datasource1')
# initialize p with full table
run.receive_data_sequenced(publisher='datasource1',
table='p',
data=[[1]],
seqnum=0,
is_snapshot=True)
actual = run.select('p(x)')
correct = 'p(1)'
self.assertTrue(helper.db_equal(actual, correct))
# add data to p
run.receive_data_sequenced(publisher='datasource1',
table='p',
data=[[[2]], []],
seqnum=1,
is_snapshot=False)
actual = run.select('p(x)')
correct = 'p(1) p(2)'
self.assertTrue(helper.db_equal(actual, correct))
# add data to q
run.receive_data_sequenced(publisher='datasource1',
table='q',
data=[[[2]], []],
seqnum=1,
is_snapshot=False)
actual = run.select('q(x)')
correct = '' # does not apply until initialize
self.assertTrue(helper.db_equal(actual, correct))
# initialize q with full table
run.receive_data_sequenced(publisher='datasource1',
table='q',
data=[[1]],
seqnum=0,
is_snapshot=True)
actual = run.select('q(x)')
correct = 'q(1) q(2)' # both initial data and preceding update applied
self.assertTrue(helper.db_equal(actual, correct))
# add data to q
run.receive_data_sequenced(publisher='datasource1',
table='q',
data=[[[3]], []],
seqnum=2,
is_snapshot=False)
actual = run.select('q(x)')
correct = 'q(1) q(2) q(3)'
self.assertTrue(helper.db_equal(actual, correct))
# add data to p
run.receive_data_sequenced(publisher='datasource1',
table='p',
data=[[[3]], []],
seqnum=2,
is_snapshot=False)
actual = run.select('p(x)')
correct = 'p(1) p(2) p(3)'
self.assertTrue(helper.db_equal(actual, correct))