def insert_rule(self, engine, statement, target=None):
statement = compile.parse1(statement)
if target is None:
e = compile.Event(statement)
else:
e = compile.Event(statement, target=target)
engine.process_policy_update([e])
def test_dependency_batch(self):
obj = self.MyObject()
run = agnostic.Runtime()
run.create_policy('test')
run.insert('p(x) :- q(x)')
run.register_trigger('p', lambda old, new: obj.increment())
rule = compile.parse1('q(x) :- r(x)')
data = compile.parse1('r(1)')
run.update([
compile.Event(rule, target='test'),
compile.Event(data, target='test')
])
self.assertEqual(obj.value, 1)
def test_initialize_tables_dse(self):
"""Test performance of initializing data with DSE and Engine.
This test populates the tables exported by a datasource driver,
and then invokes the poll() method to send that data to the
policy engine. It tests the amount of time to send tables
across the DSE and load them into the policy engine.
"""
MAX_TUPLES = 700
# install datasource driver we can control
kwds = {}
kwds['name'] = 'data'
kwds['args'] = helper.datasource_openstack_args()
kwds['driver'] = 'performance'
driver = self.cage.create_datasource_service(kwds)
self.cage.register_service(driver)
driver.poll_time = 0
self.engine.create_policy('data')
# set return value for datasource driver
facts = [(1, 2.3, 'foo', 'bar', i, 'a' * 100 + str(i))
for i in range(MAX_TUPLES)]
driver.state = {'p': facts}
# Send formula to engine (so engine subscribes to data:p)
policy = self.engine.DEFAULT_THEORY
formula = compile.parse1(
'q(1) :- data:p(1, 2.3, "foo", "bar", 1, %s)' % ('a' * 100 + '1'))
self.engine.process_policy_update(
[compile.Event(formula, target=policy)])
# Poll data and wait til it arrives at engine
driver.poll()
self.wait_til_query_nonempty('q(1)', policy)
def process_new_bindings(self, bindings, atom, insert, original_rule):
"""Process new bindings.
For each of BINDINGS, apply to ATOM, and enqueue it as an insert if
INSERT is True and as a delete otherwise.
"""
# for each binding, compute generated tuple and group bindings
# by the tuple they generated
new_atoms = {}
for binding in bindings:
new_atom = atom.plug(binding)
if new_atom not in new_atoms:
new_atoms[new_atom] = []
new_atoms[new_atom].append(
database.Database.Proof(binding, original_rule))
self.log(atom.table.table, "new tuples generated: %s",
utility.iterstr(new_atoms))
# enqueue each distinct generated tuple, recording appropriate bindings
for new_atom in new_atoms:
# self.log(event.table, "new_tuple %s: %s", new_tuple,
# new_tuples[new_tuple])
# Only enqueue if new data.
# Putting the check here is necessary to support recursion.
self.enqueue(
compile.Event(formula=new_atom,
proofs=new_atoms[new_atom],
insert=insert))
def enqueue_any(self, event):
"""Enqueue event.
Processing rules is a bit different than processing atoms
in that they generate additional events that we want
to process either before the rule is deleted or after
it is inserted. PROCESS_QUEUE is similar but assumes
that only the data will cause propagations (and ignores
included theories).
"""
# Note: all included theories must define MODIFY
formula = event.formula
if formula.is_atom():
self.log(formula.tablename(), "compute/enq: atom %s", formula)
assert not self.is_view(
formula.table.table), ("Cannot directly modify tables" +
" computed from other tables")
# self.log(formula.table, "%s: %s", text, formula)
self.enqueue(event)
return []
else:
# rules do not need to talk to included theories because they
# only generate events for views
# need to eliminate self-joins here so that we fill all
# the tables introduced by self-join elimination.
for rule in DeltaRuleTheory.eliminate_self_joins([formula]):
new_event = compile.Event(formula=rule,
insert=event.insert,
target=event.target)
self.enqueue(new_event)
return []
def receive_data_update(self, msg):
"""Handler for when dataservice publishes a delta."""
LOG.info("%s:: received update data msg for %s: %s", self.name,
msg.header['dataindex'],
";".join(str(x) for x in msg.body.data))
new_events = []
for event in msg.body.data:
assert compile.is_atom(
event.formula), ("receive_data_update received non-atom: " +
str(event.formula))
# prefix tablename with data source
actual_table = compile.Tablename.build_service_table(
msg.replyTo, event.formula.table.table)
values = [term.name for term in event.formula.arguments]
newevent = compile.Event(compile.Fact(actual_table, values),
insert=event.insert)
new_events.append(newevent)
(permitted, changes) = self.policy.update(new_events)
if not permitted:
raise exception.CongressException("Update not permitted." +
'\n'.join(
str(x) for x in changes))
else:
tablename = msg.header['dataindex']
service = msg.replyTo
LOG.debug(
"update data msg for %s from %s caused %d "
"changes: %s", tablename, service, len(changes),
";".join(str(x) for x in changes))
def test_old_new_correctness(self):
obj = self.MyObject()
run = agnostic.Runtime()
run.create_policy('test')
run.insert('p(x) :- q(x)')
run.insert('q(x) :- r(x), not s(x)')
run.insert('r(1) r(2) r(3)')
run.insert('s(2)')
oldp = set(compile.parse('p(1) p(3)'))
newp = set(compile.parse('p(1) p(2)'))
run.register_trigger('p',
lambda old, new: obj.equal(oldp, newp, old, new))
run.update([
compile.Event(compile.parse1('s(3)')),
compile.Event(compile.parse1('s(2)'), insert=False)
])
self.assertEqual(obj.equals, True)
def test_batch_change(self):
obj = self.MyObject()
run = agnostic.Runtime()
run.create_policy('test')
run.register_trigger('p', lambda old, new: obj.increment())
p1 = compile.parse1('p(1)')
result = run.update([compile.Event(p1, target='test')])
self.assertTrue(result[0], ("Update failed with errors: " +
";".join(str(x) for x in result[1])))
self.assertEqual(obj.value, 1)
def test_initialize_tables_full(self):
"""Test performance of initializing data with Datasource, DSE, Engine.
This test gives a datasource driver the Python data that would
have resulted from making an API call and parsing it into Python
and then polls that datasource, waiting until the data arrives
in the policy engine. It tests the amount of time required to
translate Python data into tables, send those tables over the DSE,
and load them into the policy engine.
"""
MAX_TUPLES = 700
# install datasource driver we can control
kwds = {}
kwds['name'] = 'data'
kwds['args'] = helper.datasource_openstack_args()
kwds['driver'] = 'performance'
driver = self.cage.create_datasource_service(kwds)
self.cage.register_service(driver)
driver.poll_time = 0
self.engine.create_policy('data')
# set return value for datasource driver
facts = [{
'field1': 1,
'field2': 2.3,
'field3': 'foo',
'field4': 'bar',
'field5': i,
'field6': 'a' * 100 + str(i)
} for i in range(MAX_TUPLES)]
driver.client_data = facts
# Send formula to engine (so engine subscribes to data:p)
policy = self.engine.DEFAULT_THEORY
formula = compile.parse1(
'q(1) :- data:p(1, 2.3, "foo", "bar", 1, %s)' % ('a' * 100 + '1'))
LOG.info("publishing rule")
self.engine.process_policy_update(
[compile.Event(formula, target=policy)])
# Poll data and wait til it arrives at engine
driver.poll()
self.wait_til_query_nonempty('q(1)', policy)
def synchronize_rules_nonlocking(self, db_session=None):
LOG.debug("Synchronizing rules on node %s", self.node.node_id)
try:
# Read rules from DB.
configured_rules = []
configured_facts = []
for r in db_policy_rules.get_policy_rules(session=db_session):
if ':-' in r.rule: # if rule has body
configured_rules.append({'rule': r.rule,
'id': r.id,
'comment': r.comment,
'name': r.name,
'policy_name': r.policy_name})
else: # head-only rule, ie., fact
configured_facts.append(
{'rule': self.engine.parse1(r.rule).pretty_str(),
# note:parse to remove effect of extraneous formatting
'policy_name': r.policy_name})
# Read rules from engine
policies = {n: self.engine.policy_object(n) for n in
self.engine.policy_names()}
active_policy_rules = []
active_policy_facts = []
for policy_name, policy in policies.items():
if policy.kind != base.DATASOURCE_POLICY_TYPE:
for active_rule in policy.content():
# FIXME: This assumes r.original_str is None iff
# r is a head-only rule (fact). This works in
# non-recursive policy but not in recursive policies
if active_rule.original_str is None:
active_policy_facts.append(
{'rule': str(active_rule.head),
'policy_name': policy_name})
else:
active_policy_rules.append(
{'rule': active_rule.original_str,
'id': active_rule.id,
'comment': active_rule.comment,
'name': active_rule.name,
'policy_name': policy_name})
# ALEX: the Rule object does not have fields like the rule-string
# or id or comment. We can add those fields to the Rule object,
# as long as we don't add them to the Fact because there are many
# fact instances. If a user tries to create a lot of Rules, they
# are probably doing something wrong and should use a datasource
# driver instead.
changes = []
# add configured rules
for r in configured_rules:
if r not in active_policy_rules:
LOG.debug("adding rule %s", str(r))
parsed_rule = self.engine.parse1(r['rule'])
parsed_rule.set_id(r['id'])
parsed_rule.set_name(r['name'])
parsed_rule.set_comment(r['comment'])
parsed_rule.set_original_str(r['rule'])
event = compile.Event(formula=parsed_rule,
insert=True,
target=r['policy_name'])
changes.append(event)
# add configured facts
for r in configured_facts:
if r not in active_policy_facts:
LOG.debug("adding rule %s", str(r))
parsed_rule = self.engine.parse1(r['rule'])
event = compile.Event(formula=parsed_rule,
insert=True,
target=r['policy_name'])
changes.append(event)
# remove active rules not configured
for r in active_policy_rules:
if r not in configured_rules:
LOG.debug("removing rule %s", str(r))
parsed_rule = self.engine.parse1(r['rule'])
parsed_rule.set_id(r['id'])
parsed_rule.set_name(r['name'])
parsed_rule.set_comment(r['comment'])
parsed_rule.set_original_str(r['rule'])
event = compile.Event(formula=parsed_rule,
insert=False,
target=r['policy_name'])
changes.append(event)
# remove active facts not configured
for r in active_policy_facts:
if r not in configured_facts:
LOG.debug("removing rule %s", str(r))
parsed_rule = self.engine.parse1(r['rule'])
event = compile.Event(formula=parsed_rule,
insert=False,
target=r['policy_name'])
changes.append(event)
permitted, changes = self.engine.process_policy_update(changes)
LOG.info("synchronize_rules, permitted %d, made %d changes on "
"node %s", permitted, len(changes), self.node.node_id)
except Exception:
LOG.exception("synchronizing rules failed")
def _create_event(self, table, tuple_, insert, target):
return compile.Event(compile.Literal.create_from_table_tuple(
table, tuple_),
insert=insert,
target=target)
def test_event_equality(self):
r1 = compile.parse1('p(x) :- q(x)')
r2 = compile.parse1('p(x) :- q(x)')
e1 = compile.Event(formula=r1, target='alice', insert=True)
e2 = compile.Event(formula=r2, target='alice', insert=True)
self.assertEqual(e1, e2)
def delete(self, atom, proofs=None):
"""Deletes ATOM from the DB. Returns changes."""
return self.modify(
compile.Event(formula=atom, insert=False, proofs=proofs))
def delete(self, formula):
return self.update([compile.Event(formula=formula, insert=False)])
def insert(self, formula):
return self.update([compile.Event(formula=formula, insert=True)])
def insert(self, atom, proofs=None):
"""Inserts ATOM into the DB. Returns changes."""
return self.modify(
compile.Event(formula=atom, insert=True, proofs=proofs))
def define(self, rules):
"""Empties and then inserts RULES."""
self.empty()
return self.update(
[compile.Event(formula=rule, insert=True) for rule in rules])
def insert(self, rule):
changes = self.update([compile.Event(formula=rule, insert=True)])
return [event.formula for event in changes]
def delete(self, rule):
changes = self.update([compile.Event(formula=rule, insert=False)])
return [event.formula for event in changes]
def test_nodes_edges(self):
g = compile.RuleDependencyGraph()
# first insertion
g.formula_insert(compile.parse1('p(x), q(x) :- r(x), s(x)'))
self.assertTrue(g.node_in('p'))
self.assertTrue(g.node_in('q'))
self.assertTrue(g.node_in('r'))
self.assertTrue(g.node_in('s'))
self.assertTrue(g.edge_in('p', 'r', False))
self.assertTrue(g.edge_in('p', 's', False))
self.assertTrue(g.edge_in('q', 'r', False))
self.assertTrue(g.edge_in('q', 's', False))
self.assertFalse(g.has_cycle())
# another insertion
g.formula_insert(compile.parse1('r(x) :- t(x)'))
self.assertTrue(g.node_in('p'))
self.assertTrue(g.node_in('q'))
self.assertTrue(g.node_in('r'))
self.assertTrue(g.node_in('s'))
self.assertTrue(g.edge_in('p', 'r', False))
self.assertTrue(g.edge_in('p', 's', False))
self.assertTrue(g.edge_in('q', 'r', False))
self.assertTrue(g.edge_in('q', 's', False))
self.assertTrue(g.node_in('t'))
self.assertTrue(g.edge_in('r', 't', False))
self.assertFalse(g.has_cycle())
# 3rd insertion, creating a cycle
g.formula_insert(compile.parse1('t(x) :- p(x)'))
self.assertTrue(g.edge_in('t', 'p', False))
self.assertTrue(g.has_cycle())
# deletion
g.formula_delete(compile.parse1('p(x), q(x) :- r(x), s(x)'))
self.assertTrue(g.node_in('p'))
self.assertTrue(g.node_in('r'))
self.assertTrue(g.node_in('t'))
self.assertTrue(g.edge_in('r', 't', False))
self.assertTrue(g.edge_in('t', 'p', False))
self.assertFalse(g.has_cycle())
# double-insertion
g.formula_insert(compile.parse1('p(x) :- q(x), r(x)'))
g.formula_insert(compile.parse1('p(1) :- r(1)'))
self.assertTrue(g.has_cycle())
# deletion -- checking for bag semantics
g.formula_delete(compile.parse1('p(1) :- r(1)'))
self.assertTrue(g.has_cycle())
g.formula_delete(compile.parse1('p(x) :- q(x), r(x)'))
self.assertFalse(g.has_cycle())
# update
g.formula_update([
compile.Event(compile.parse1('a(x) :- b(x)')),
compile.Event(compile.parse1('b(x) :- c(x)')),
compile.Event(compile.parse1('c(x) :- a(x)'))
])
self.assertTrue(g.has_cycle())
g.formula_update(
[compile.Event(compile.parse1('c(x) :- a(x)'), insert=False)])
self.assertFalse(g.has_cycle())
# cycle enumeration
g = compile.RuleDependencyGraph()
g.formula_insert(compile.parse1('p(x) :- q(x), r(x)'))
g.formula_insert(compile.parse1('q(x) :- t(x), not s(x)'))
g.formula_insert(compile.parse1('t(x) :- t(x), p(x), q(x)'))
self.assertTrue(g.has_cycle())
self.assertEqual(len(g.cycles()), 3)
expected_cycle_set = set([
utility.Cycle(['p', 'q', 't', 'p']),
utility.Cycle(['q', 't', 'q']),
utility.Cycle(['t', 't'])
])
actual_cycle_set = set([
utility.Cycle(g.cycles()[0]),
utility.Cycle(g.cycles()[1]),
utility.Cycle(g.cycles()[2])
])
self.assertEqual(expected_cycle_set, actual_cycle_set)
def test_complex_dependency(self):
g = compile.RuleDependencyGraph()
reg = agnostic.TriggerRegistry(g)
g.formula_insert(compile.parse1('p(x) :- q(x)'), 'alice')
g.formula_insert(compile.parse1('q(x) :- r(x), s(x)'), 'alice')
g.formula_insert(compile.parse1('r(x) :- t(x, y), u(y)'), 'alice')
g.formula_insert(compile.parse1('separate(x) :- separate2(x)'),
'alice')
g.formula_insert(compile.parse1('notrig(x) :- notrig2(x)'), 'alice')
p_trigger = reg.register_table('p', 'alice', self.f)
sep_trigger = reg.register_table('separate', 'alice', self.f)
# individual tables
self.assertEqual(reg.relevant_triggers(['alice:p']), set([p_trigger]))
self.assertEqual(reg.relevant_triggers(['alice:q']), set([p_trigger]))
self.assertEqual(reg.relevant_triggers(['alice:r']), set([p_trigger]))
self.assertEqual(reg.relevant_triggers(['alice:s']), set([p_trigger]))
self.assertEqual(reg.relevant_triggers(['alice:t']), set([p_trigger]))
self.assertEqual(reg.relevant_triggers(['alice:u']), set([p_trigger]))
self.assertEqual(reg.relevant_triggers(['alice:notrig']), set())
self.assertEqual(reg.relevant_triggers(['alice:notrig2']), set([]))
self.assertEqual(reg.relevant_triggers(['alice:separate']),
set([sep_trigger]))
self.assertEqual(reg.relevant_triggers(['alice:separate2']),
set([sep_trigger]))
# groups of tables
self.assertEqual(reg.relevant_triggers(['alice:p', 'alice:q']),
set([p_trigger]))
self.assertEqual(reg.relevant_triggers(['alice:separate', 'alice:p']),
set([p_trigger, sep_trigger]))
self.assertEqual(reg.relevant_triggers(['alice:notrig', 'alice:p']),
set([p_trigger]))
# events: data
event = compile.Event(compile.parse1('q(1)'), target='alice')
self.assertEqual(reg.relevant_triggers([event]), set([p_trigger]))
event = compile.Event(compile.parse1('u(1)'), target='alice')
self.assertEqual(reg.relevant_triggers([event]), set([p_trigger]))
event = compile.Event(compile.parse1('separate2(1)'), target='alice')
self.assertEqual(reg.relevant_triggers([event]), set([sep_trigger]))
event = compile.Event(compile.parse1('notrig2(1)'), target='alice')
self.assertEqual(reg.relevant_triggers([event]), set([]))
# events: rules
event = compile.Event(compile.parse1('separate(x) :- q(x)'),
target='alice')
self.assertEqual(reg.relevant_triggers([event]), set([sep_trigger]))
event = compile.Event(compile.parse1('notrig(x) :- q(x)'),
target='alice')
self.assertEqual(reg.relevant_triggers([event]), set([]))
event = compile.Event(compile.parse1('r(x) :- q(x)'), target='alice')
self.assertEqual(reg.relevant_triggers([event]), set([p_trigger]))
# events: multiple rules and data
event1 = compile.Event(compile.parse1('r(x) :- q(x)'), target='alice')
event2 = compile.Event(compile.parse1('separate2(1)'), target='alice')
self.assertEqual(reg.relevant_triggers([event1, event2]),
set([p_trigger, sep_trigger]))
event1 = compile.Event(compile.parse1('r(x) :- q(x)'), target='alice')
event2 = compile.Event(compile.parse1('notrigger2(1)'), target='alice')
self.assertEqual(reg.relevant_triggers([event1, event2]),
set([p_trigger]))
