def prepush_processor(self, data, dataindex, type=None):
"""Called before push.
Takes as input the DATA that the receiver needs and returns
the payload for the message. If this is a regular publication
message, make the payload just the delta; otherwise, make the
payload the entire table.
"""
# This routine basically ignores DATA and sends a delta
# of the self.prior_state and self.state, for the DATAINDEX
# part of the state.
self.log("prepush_processor: dataindex <%s> data: %s", dataindex, data)
# if not a regular publication, just return the original data
if type != 'pub':
self.log("prepush_processor: returned original data")
if type == 'sub':
# Always want to send initialization of []
if data is None:
return []
else:
return data
return data
# grab deltas
to_add = self.state_set_diff(self.state, self.prior_state, dataindex)
to_del = self.state_set_diff(self.prior_state, self.state, dataindex)
self.log("to_add: %s", to_add)
self.log("to_del: %s", to_del)
# create Events
result = []
for row in to_add:
formula = compile.Literal.create_from_table_tuple(dataindex, row)
event = runtime.Event(formula=formula, insert=True)
result.append(event)
for row in to_del:
formula = compile.Literal.create_from_table_tuple(dataindex, row)
event = runtime.Event(formula=formula, insert=False)
result.append(event)
if len(result) == 0:
# Policy engine expects an empty update to be an init msg
# So if delta is empty, return None, which signals
# the message should not be sent.
result = None
text = "None"
else:
text = runtime.iterstr(result)
self.log("prepush_processor for <%s> returning with %s items",
dataindex, text)
return result
def test_dependency_graph(self):
"""Test that dependency graph gets updated correctly."""
run = runtime.Runtime()
run.debug_mode()
g = run.global_dependency_graph
run.create_policy('test')
run.insert('p(x) :- q(x), nova:q(x)', target='test')
self.assertTrue(g.edge_in('test:p', 'nova:q', False))
self.assertTrue(g.edge_in('test:p', 'test:q', False))
run.insert('p(x) :- s(x)', target='test')
self.assertTrue(g.edge_in('test:p', 'nova:q', False))
self.assertTrue(g.edge_in('test:p', 'test:q', False))
self.assertTrue(g.edge_in('test:p', 'test:s', False))
run.insert('q(x) :- nova:r(x)', target='test')
self.assertTrue(g.edge_in('test:p', 'nova:q', False))
self.assertTrue(g.edge_in('test:p', 'test:q', False))
self.assertTrue(g.edge_in('test:p', 'test:s', False))
self.assertTrue(g.edge_in('test:q', 'nova:r', False))
run.delete('p(x) :- q(x), nova:q(x)', target='test')
self.assertTrue(g.edge_in('test:p', 'test:s', False))
self.assertTrue(g.edge_in('test:q', 'nova:r', False))
run.update([
runtime.Event(helper.str2form('p(x) :- q(x), nova:q(x)'),
target='test')
])
self.assertTrue(g.edge_in('test:p', 'nova:q', False))
self.assertTrue(g.edge_in('test:p', 'test:q', False))
self.assertTrue(g.edge_in('test:p', 'test:s', False))
self.assertTrue(g.edge_in('test:q', 'nova:r', False))
def test_policy_data(self):
"""Test policy properly inserts data and processes it normally."""
cage = congress.dse.d6cage.d6Cage()
cage.loadModule(
"TestDriver",
helper.data_module_path("../tests/datasources/test_driver.py"))
cage.loadModule("TestPolicy", helper.policy_module_path())
cage.createservice(name="data",
moduleName="TestDriver",
args=helper.datasource_openstack_args())
cage.createservice(name="policy",
moduleName="TestPolicy",
args={
'd6cage': cage,
'rootdir': ''
})
data = cage.services['data']['object']
policy = cage.services['policy']['object']
# turn off module-schema syntax checking
policy.create_policy('data')
policy.set_schema('data', compile.Schema({'p': (1, )}))
policy.subscribe('data', 'p', callback=policy.receive_data)
formula = policy.parse1('p(1)')
# sending a single Insert. (Default for Event is Insert.)
data.publish('p', [runtime.Event(formula)])
helper.retry_check_db_equal(policy, 'data:p(x)', 'data:p(1)')
def test_policy_tables(self):
"""Test basic DSE functionality with policy engine and the API."""
cage = congress.dse.d6cage.d6Cage()
cage.loadModule(
"TestDriver",
helper.data_module_path("../tests/datasources/test_driver.py"))
cage.loadModule("TestPolicy", helper.policy_module_path())
cage.createservice(name="data",
moduleName="TestDriver",
args=helper.datasource_openstack_args())
# using regular testdriver as API for now
cage.createservice(name="api",
moduleName="TestDriver",
args=helper.datasource_openstack_args())
cage.createservice(name="policy",
moduleName="TestPolicy",
args={
'd6cage': cage,
'rootdir': ''
})
data = cage.services['data']['object']
api = cage.services['api']['object']
policy = cage.services['policy']['object']
policy.create_policy('data')
policy.set_schema('data', compile.Schema({'q': (1, )}))
policy.subscribe('api',
'policy-update',
callback=policy.receive_policy_update)
# simulate API call for insertion of policy statements
formula = policy.parse1('p(x) :- data:q(x)')
api.publish('policy-update', [runtime.Event(formula)])
helper.retry_check_nonempty_last_policy_change(policy)
# simulate data source publishing to q
formula = policy.parse1('q(1)')
data.publish('q', [runtime.Event(formula)])
helper.retry_check_db_equal(policy, 'data:q(x)', 'data:q(1)')
# check that policy did the right thing with data
e = helper.db_equal(policy.select('p(x)'), 'p(1)')
self.assertTrue(e, 'Policy insert')
# check that publishing into 'p' does not work
formula = policy.parse1('p(3)')
data.publish('p', [runtime.Event(formula)])
# can't actually check that the update for p does not arrive
# so instead wait a bit and check
helper.pause()
e = helper.db_equal(policy.select('p(x)'), 'p(1)')
self.assertTrue(e, 'Policy non-insert')
def change_rule(self, parsed_rule, context, insert=True):
policy_name = self.policy_name(context)
if policy_name not in self.engine.theory:
raise KeyError("Policy with ID '%s' does not exist", policy_name)
event = runtime.Event(formula=parsed_rule,
insert=insert,
target=policy_name)
(permitted, changes) = self.engine.process_policy_update([event])
if not permitted:
raise compile.CongressException("Errors: " +
";".join((str(x)
for x in changes)))
return changes
def test_multi_policy_update(self):
"""Test updates that apply to multiple policies."""
def check_equal(actual, correct):
e = helper.datalog_equal(actual, correct)
self.assertTrue(e)
run = runtime.Runtime()
run.create_policy('th1')
run.create_policy('th2')
events1 = [
runtime.Event(formula=x, insert=True, target='th1')
for x in helper.str2pol("p(1) p(2) q(1) q(3)")
]
events2 = [
runtime.Event(formula=x, insert=True, target='th2')
for x in helper.str2pol("r(1) r(2) t(1) t(4)")
]
run.update(events1 + events2)
check_equal(run.select('p(x)', 'th1'), 'p(1) p(2)')
check_equal(run.select('q(x)', 'th1'), 'q(1) q(3)')
check_equal(run.select('r(x)', 'th2'), 'r(1) r(2)')
check_equal(run.select('t(x)', 'th2'), 't(1) t(4)')
def test_policy_subscriptions(self):
"""Test that policy engine subscriptions adjust to policy changes."""
engine = self.engine
api = self.api
cage = self.cage
policy = engine.DEFAULT_THEORY
# Send formula
formula = test_neutron.create_network_group('p')
LOG.debug("Sending formula: %s", formula)
api['rule'].publish('policy-update',
[runtime.Event(formula, target=policy)])
# check we have the proper subscriptions
self.assertTrue('neutron' in cage.services)
neutron = cage.service_object('neutron')
helper.retry_check_subscriptions(engine, [('neutron', 'networks')])
helper.retry_check_subscribers(neutron, [(engine.name, 'networks')])
def test_neutron(self):
"""Test polling and publishing of neutron updates."""
engine = self.engine
api = self.api
cage = self.cage
policy = engine.DEFAULT_THEORY
# Send formula
formula = test_neutron.create_network_group('p')
LOG.debug("Sending formula: %s", formula)
api['rule'].publish('policy-update',
[runtime.Event(formula, target=policy)])
helper.retry_check_nonempty_last_policy_change(engine)
LOG.debug("All services: %s", cage.services.keys())
neutron = cage.service_object('neutron')
neutron.poll()
ans = ('p("240ff9df-df35-43ae-9df5-27fae87f2492") ')
helper.retry_check_db_equal(engine, 'p(x)', ans, target=policy)
def test_multiple(self):
"""Test polling and publishing of multiple neutron instances."""
api = self.api
cage = self.cage
engine = self.engine
policy = engine.DEFAULT_THEORY
# Send formula
formula = test_neutron.create_networkXnetwork_group('p')
api['rule'].publish('policy-update',
[runtime.Event(formula, target=policy)])
helper.retry_check_nonempty_last_policy_change(engine)
# poll datasources
neutron = cage.service_object('neutron')
neutron2 = cage.service_object('neutron2')
neutron.poll()
neutron2.poll()
# check answer
ans = ('p("240ff9df-df35-43ae-9df5-27fae87f2492", '
' "240ff9df-df35-43ae-9df5-27fae87f2492") ')
helper.retry_check_db_equal(engine, 'p(x,y)', ans, target=policy)
def benchmark_datasource_to_policy_update(self, size):
"""Benchmark small datsource update to policy propagation.
Time the propagation of a datasource update from datasource.poll() to
completion of a simple policy update.
"""
LOG.info("%s:: benchmarking datasource update of %d rows", size)
self.datasource.datarows = size
table_name = self.table_name
# dummy policy only intended to produce a subscriber for the table
key_to_index = self.datasource.get_column_map(table_name)
id_index = 'x%d' % key_to_index.items()[0][1]
max_index = max(key_to_index.values())
args = ['x%d' % i for i in xrange(max_index + 1)]
formula = compile.parse1('p(%s) :- benchmark:%s(%s)' % (id_index,
table_name, ','.join(args)))
# publish the formula and verify we see a subscription
LOG.debug('%s:: sending formula: %s', self.__class__.__name__, formula)
self.api['rule'].publish('policy-update', [runtime.Event(formula)])
helper.retry_check_subscriptions(
self.engine, [('benchmark', table_name)])
helper.retry_check_subscribers(
self.datasource, [(self.engine.name, table_name)])
# intercept inbox.task_done() so we know when it's finished. Sadly,
# eventlet doesn't have a condition-like object.
fake_condition = eventlet.Queue()
fake_notify = functools.partial(fake_condition.put_nowait, True)
self.mox.StubOutWithMock(self.engine.inbox, "task_done")
self.engine.inbox.task_done().WithSideEffects(fake_notify)
self.mox.ReplayAll()
LOG.info("%s:: polling datasource", self.__class__.__name__)
self.datasource.poll()
fake_condition.get(timeout=30)
self.mox.VerifyAll()
def _create_event(self, table, tuple_, insert, target):
return runtime.Event(Literal.create_from_table_tuple(table, tuple_),
insert=insert, target=target)
