def prepush_processor(self, data, dataindex, type=None):
"""Called before push.
Takes as input the DATA that the receiver needs and returns
the payload for the message. If this is a regular publication
message, make the payload just the delta; otherwise, make the
payload the entire table.
"""
# This routine basically ignores DATA and sends a delta
# of the self.prior_state and self.state, for the DATAINDEX
# part of the state.
self.log("prepush_processor: dataindex <%s> data: %s", dataindex, data)
# if not a regular publication, just return the original data
if type != 'pub':
self.log("prepush_processor: returned original data")
if type == 'sub':
# Always want to send initialization of []
if data is None:
return []
else:
return data
return data
# grab deltas
to_add = self.state_set_diff(self.state, self.prior_state, dataindex)
to_del = self.state_set_diff(self.prior_state, self.state, dataindex)
self.log("to_add: %s", to_add)
self.log("to_del: %s", to_del)
# create Events
result = []
for row in to_add:
formula = compile.Literal.create_from_table_tuple(dataindex, row)
event = agnostic.Event(formula=formula, insert=True)
result.append(event)
for row in to_del:
formula = compile.Literal.create_from_table_tuple(dataindex, row)
event = agnostic.Event(formula=formula, insert=False)
result.append(event)
if len(result) == 0:
# Policy engine expects an empty update to be an init msg
# So if delta is empty, return None, which signals
# the message should not be sent.
result = None
text = "None"
else:
text = agnostic.iterstr(result)
self.log("prepush_processor for <%s> returning with %s items",
dataindex, text)
return result
def test_initialize_tables_dse(self):
"""Test performance of initializing data with DSE and Engine.
This test populates the tables exported by a datasource driver,
and then invokes the poll() method to send that data to the
policy engine. It tests the amount of time to send tables
across the DSE and load them into the policy engine.
"""
MAX_TUPLES = 700
# install datasource driver we can control
self.cage.loadModule(
"TestDriver",
helper.data_module_path("../tests/datasources/test_driver.py"))
self.cage.createservice(name="data",
moduleName="TestDriver",
args=helper.datasource_openstack_args())
driver = self.cage.service_object('data')
driver.poll_time = 0
self.engine.create_policy('data')
# set return value for datasource driver
facts = [(1, 2.3, 'foo', 'bar', i, 'a' * 100 + str(i))
for i in range(MAX_TUPLES)]
driver.state = {'p': facts}
# Send formula to engine (so engine subscribes to data:p)
policy = self.engine.DEFAULT_THEORY
formula = compile.parse1(
'q(1) :- data:p(1, 2.3, "foo", "bar", 1, %s)' % ('a' * 100 + '1'))
self.api['rule'].publish('policy-update',
[agnostic.Event(formula, target=policy)])
# Poll data and wait til it arrives at engine
driver.poll()
self.wait_til_query_nonempty('q(1)', policy)
def test_policy_data(self):
"""Test policy properly inserts data and processes it normally."""
cage = congress.dse.d6cage.d6Cage()
cage.loadModule(
"TestDriver",
helper.data_module_path("../tests/datasources/test_driver.py"))
cage.loadModule("TestPolicy", helper.policy_module_path())
cage.createservice(name="data",
moduleName="TestDriver",
args=helper.datasource_openstack_args())
cage.createservice(name="policy",
moduleName="TestPolicy",
args={
'd6cage': cage,
'rootdir': ''
})
data = cage.services['data']['object']
policy = cage.services['policy']['object']
# turn off module-schema syntax checking
policy.create_policy('data')
policy.set_schema('data', compile.Schema({'p': (1, )}))
policy.subscribe('data', 'p', callback=policy.receive_data)
formula = policy.parse1('p(1)')
# sending a single Insert. (Default for Event is Insert.)
data.publish('p', [agnostic.Event(formula)])
helper.retry_check_db_equal(policy, 'data:p(x)', 'data:p(1)')
def test_dependency_graph(self):
"""Test that dependency graph gets updated correctly."""
run = agnostic.Runtime()
run.debug_mode()
g = run.global_dependency_graph
run.create_policy('test')
run.insert('p(x) :- q(x), nova:q(x)', target='test')
self.assertTrue(g.edge_in('test:p', 'nova:q', False))
self.assertTrue(g.edge_in('test:p', 'test:q', False))
run.insert('p(x) :- s(x)', target='test')
self.assertTrue(g.edge_in('test:p', 'nova:q', False))
self.assertTrue(g.edge_in('test:p', 'test:q', False))
self.assertTrue(g.edge_in('test:p', 'test:s', False))
run.insert('q(x) :- nova:r(x)', target='test')
self.assertTrue(g.edge_in('test:p', 'nova:q', False))
self.assertTrue(g.edge_in('test:p', 'test:q', False))
self.assertTrue(g.edge_in('test:p', 'test:s', False))
self.assertTrue(g.edge_in('test:q', 'nova:r', False))
run.delete('p(x) :- q(x), nova:q(x)', target='test')
self.assertTrue(g.edge_in('test:p', 'test:s', False))
self.assertTrue(g.edge_in('test:q', 'nova:r', False))
run.update([
agnostic.Event(helper.str2form('p(x) :- q(x), nova:q(x)'),
target='test')
])
self.assertTrue(g.edge_in('test:p', 'nova:q', False))
self.assertTrue(g.edge_in('test:p', 'test:q', False))
self.assertTrue(g.edge_in('test:p', 'test:s', False))
self.assertTrue(g.edge_in('test:q', 'nova:r', False))
def test_policy_tables(self):
"""Test basic DSE functionality with policy engine and the API."""
cage = congress.dse.d6cage.d6Cage()
cage.loadModule(
"TestDriver",
helper.data_module_path("../tests/datasources/test_driver.py"))
cage.loadModule("TestPolicy", helper.policy_module_path())
cage.createservice(name="data",
moduleName="TestDriver",
args=helper.datasource_openstack_args())
# using regular testdriver as API for now
cage.createservice(name="api",
moduleName="TestDriver",
args=helper.datasource_openstack_args())
cage.createservice(name="policy",
moduleName="TestPolicy",
args={
'd6cage': cage,
'rootdir': ''
})
data = cage.services['data']['object']
api = cage.services['api']['object']
policy = cage.services['policy']['object']
policy.create_policy('data')
policy.set_schema('data', compile.Schema({'q': (1, )}))
policy.subscribe('api',
'policy-update',
callback=policy.receive_policy_update)
# simulate API call for insertion of policy statements
formula = policy.parse1('p(x) :- data:q(x)')
api.publish('policy-update', [agnostic.Event(formula)])
helper.retry_check_nonempty_last_policy_change(policy)
# simulate data source publishing to q
formula = policy.parse1('q(1)')
data.publish('q', [agnostic.Event(formula)])
helper.retry_check_db_equal(policy, 'data:q(x)', 'data:q(1)')
# check that policy did the right thing with data
e = helper.db_equal(policy.select('p(x)'), 'p(1)')
self.assertTrue(e, 'Policy insert')
# check that publishing into 'p' does not work
formula = policy.parse1('p(3)')
data.publish('p', [agnostic.Event(formula)])
# can't actually check that the update for p does not arrive
# so instead wait a bit and check
helper.pause()
e = helper.db_equal(policy.select('p(x)'), 'p(1)')
self.assertTrue(e, 'Policy non-insert')
def change_rule(self, parsed_rule, context, insert=True):
policy_name = self.policy_name(context)
if policy_name not in self.engine.theory:
raise KeyError("Policy with ID '%s' does not exist", policy_name)
event = agnostic.Event(formula=parsed_rule,
insert=insert,
target=policy_name)
(permitted, changes) = self.engine.process_policy_update([event])
if not permitted:
raise PolicyException("Errors: " + ";".join((str(x)
for x in changes)))
return changes
def test_multi_policy_update(self):
"""Test updates that apply to multiple policies."""
def check_equal(actual, correct):
e = helper.datalog_equal(actual, correct)
self.assertTrue(e)
run = agnostic.Runtime()
run.create_policy('th1')
run.create_policy('th2')
events1 = [
agnostic.Event(formula=x, insert=True, target='th1')
for x in helper.str2pol("p(1) p(2) q(1) q(3)")
]
events2 = [
agnostic.Event(formula=x, insert=True, target='th2')
for x in helper.str2pol("r(1) r(2) t(1) t(4)")
]
run.update(events1 + events2)
check_equal(run.select('p(x)', 'th1'), 'p(1) p(2)')
check_equal(run.select('q(x)', 'th1'), 'q(1) q(3)')
check_equal(run.select('r(x)', 'th2'), 'r(1) r(2)')
check_equal(run.select('t(x)', 'th2'), 't(1) t(4)')
def test_policy_subscriptions(self):
"""Test that policy engine subscriptions adjust to policy changes."""
engine = self.engine
api = self.api
cage = self.cage
policy = engine.DEFAULT_THEORY
# Send formula
formula = test_neutron.create_network_group('p')
LOG.debug("Sending formula: %s", formula)
api['rule'].publish('policy-update',
[agnostic.Event(formula, target=policy)])
# check we have the proper subscriptions
self.assertTrue('neutron' in cage.services)
neutron = cage.service_object('neutron')
helper.retry_check_subscriptions(engine, [('neutron', 'networks')])
helper.retry_check_subscribers(neutron, [(engine.name, 'networks')])
def test_initialize_tables_full(self):
"""Test performance of initializing data with Datasource, DSE, Engine.
This test gives a datasource driver the Python data that would
have resulted from making an API call and parsing it into Python
and then polls that datasource, waiting until the data arrives
in the policy engine. It tests the amount of time required to
translate Python data into tables, send those tables over the DSE,
and load them into the policy engine.
"""
MAX_TUPLES = 700
# install datasource driver we can control
self.cage.loadModule(
"PerformanceTestDriver",
helper.data_module_path(
"../tests/datasources/performance_datasource_driver.py"))
self.cage.createservice(name="data",
moduleName="PerformanceTestDriver",
args=helper.datasource_openstack_args())
driver = self.cage.service_object('data')
driver.poll_time = 0
self.engine.create_policy('data')
# set return value for datasource driver
facts = [{
'field1': 1,
'field2': 2.3,
'field3': 'foo',
'field4': 'bar',
'field5': i,
'field6': 'a' * 100 + str(i)
} for i in range(MAX_TUPLES)]
driver.client_data = facts
# Send formula to engine (so engine subscribes to data:p)
policy = self.engine.DEFAULT_THEORY
formula = compile.parse1(
'q(1) :- data:p(1, 2.3, "foo", "bar", 1, %s)' % ('a' * 100 + '1'))
LOG.info("publishing rule")
self.api['rule'].publish('policy-update',
[agnostic.Event(formula, target=policy)])
# Poll data and wait til it arrives at engine
driver.poll()
self.wait_til_query_nonempty('q(1)', policy)
def test_neutron(self):
"""Test polling and publishing of neutron updates."""
engine = self.engine
api = self.api
cage = self.cage
policy = engine.DEFAULT_THEORY
# Send formula
formula = test_neutron.create_network_group('p')
LOG.debug("Sending formula: %s", formula)
api['rule'].publish('policy-update',
[agnostic.Event(formula, target=policy)])
helper.retry_check_nonempty_last_policy_change(engine)
LOG.debug("All services: %s", cage.services.keys())
neutron = cage.service_object('neutron')
neutron.poll()
ans = ('p("240ff9df-df35-43ae-9df5-27fae87f2492") ')
helper.retry_check_db_equal(engine, 'p(x)', ans, target=policy)
def test_multiple(self):
"""Test polling and publishing of multiple neutron instances."""
api = self.api
cage = self.cage
engine = self.engine
policy = engine.DEFAULT_THEORY
# Send formula
formula = test_neutron.create_networkXnetwork_group('p')
api['rule'].publish('policy-update',
[agnostic.Event(formula, target=policy)])
helper.retry_check_nonempty_last_policy_change(engine)
# poll datasources
neutron = cage.service_object('neutron')
neutron2 = cage.service_object('neutron2')
neutron.poll()
neutron2.poll()
# check answer
ans = ('p("240ff9df-df35-43ae-9df5-27fae87f2492", '
' "240ff9df-df35-43ae-9df5-27fae87f2492") ')
helper.retry_check_db_equal(engine, 'p(x,y)', ans, target=policy)
def benchmark_datasource_to_policy_update(self, size):
"""Benchmark small datsource update to policy propagation.
Time the propagation of a datasource update from datasource.poll() to
completion of a simple policy update.
"""
LOG.info("%s:: benchmarking datasource update of %d rows", size)
self.datasource.datarows = size
table_name = self.table_name
# dummy policy only intended to produce a subscriber for the table
key_to_index = self.datasource.get_column_map(table_name)
id_index = 'x%d' % key_to_index.items()[0][1]
max_index = max(key_to_index.values())
args = ['x%d' % i for i in xrange(max_index + 1)]
formula = compile.parse1('p(%s) :- benchmark:%s(%s)' %
(id_index, table_name, ','.join(args)))
# publish the formula and verify we see a subscription
LOG.debug('%s:: sending formula: %s', self.__class__.__name__, formula)
self.api['rule'].publish('policy-update', [agnostic.Event(formula)])
helper.retry_check_subscriptions(self.engine,
[('benchmark', table_name)])
helper.retry_check_subscribers(self.datasource,
[(self.engine.name, table_name)])
# intercept inbox.task_done() so we know when it's finished. Sadly,
# eventlet doesn't have a condition-like object.
fake_condition = eventlet.Queue()
fake_notify = functools.partial(fake_condition.put_nowait, True)
self.mox.StubOutWithMock(self.engine.inbox, "task_done")
self.engine.inbox.task_done().WithSideEffects(fake_notify)
self.mox.ReplayAll()
LOG.info("%s:: polling datasource", self.__class__.__name__)
self.datasource.poll()
fake_condition.get(timeout=30)
self.mox.VerifyAll()
def _create_event(self, table, tuple_, insert, target):
return agnostic.Event(Literal.create_from_table_tuple(table, tuple_),
insert=insert,
target=target)