def process_queue(self):
"""Data and rule propagation routine.
Returns list of events that were not noops
"""
self.log(None, "Processing queue")
history = []
while len(self.queue) > 0:
event = self.queue.dequeue()
self.log(event.tablename(), "Dequeued %s", event)
if compile.is_regular_rule(event.formula):
changes = self.delta_rules.modify(event)
if len(changes) > 0:
history.extend(changes)
bindings = self.top_down_evaluation(
event.formula.variables(), event.formula.body)
self.log(event.formula.tablename(),
"new bindings after top-down: %s",
iterstr(bindings))
self.process_new_bindings(bindings, event.formula.head,
event.insert, event.formula)
else:
self.propagate(event)
history.extend(self.database.modify(event))
self.log(event.tablename(), "History: %s", iterstr(history))
return history
def process_queue(self):
"""Data and rule propagation routine.
Returns list of events that were not noops
"""
self.log(None, "Processing queue")
history = []
while len(self.queue) > 0:
event = self.queue.dequeue()
self.log(event.tablename(), "Dequeued %s", event)
if compile.is_regular_rule(event.formula):
changes = self.delta_rules.modify(event)
if len(changes) > 0:
history.extend(changes)
bindings = self.top_down_evaluation(
event.formula.variables(), event.formula.body)
self.log(event.formula.tablename(),
"new bindings after top-down: %s",
utility.iterstr(bindings))
self.process_new_bindings(bindings, event.formula.head,
event.insert, event.formula)
else:
self.propagate(event)
history.extend(self.database.modify(event))
self.log(event.tablename(), "History: %s",
utility.iterstr(history))
return history
def __str__(self):
return (
"TopDownCaller<variables={}, binding={}, find_all={}, "
"results={}, save={}, support={}>".format(
utility.iterstr(self.variables), str(self.binding),
str(self.find_all), utility.iterstr(self.results),
repr(self.save), utility.iterstr(self.support)))
def create(ac_code, class_code):
run = self.prep_runtime()
acth = run.ACCESSCONTROL_THEORY
permitted, errors = run.insert(ac_code, target=acth)
self.assertTrue(permitted,
"Error in access control policy: {}".format(
utility.iterstr(errors)))
clsth = run.CLASSIFY_THEORY
permitted, errors = run.insert(class_code, target=clsth)
self.assertTrue(permitted, "Error in classifier policy: {}".format(
utility.iterstr(errors)))
return run
def update(self, events):
"""Apply EVENTS.
And return the list of EVENTS that actually
changed the theory. Each event is the insert or delete of
a policy statement.
"""
changes = []
self.log(None, "Update %s", utility.iterstr(events))
try:
for event in events:
schema_changes = self.update_rule_schema(
event.formula, event.insert)
formula = compile.reorder_for_safety(event.formula)
if event.insert:
if self._insert_actual(formula):
changes.append(event)
else:
self.revert_schema(schema_changes)
else:
if self._delete_actual(formula):
changes.append(event)
else:
self.revert_schema(schema_changes)
except Exception:
LOG.exception("runtime caught an exception")
raise
return changes
def process_new_bindings(self, bindings, atom, insert, original_rule):
"""Process new bindings.
For each of BINDINGS, apply to ATOM, and enqueue it as an insert if
INSERT is True and as a delete otherwise.
"""
# for each binding, compute generated tuple and group bindings
# by the tuple they generated
new_atoms = {}
for binding in bindings:
new_atom = atom.plug(binding)
if new_atom not in new_atoms:
new_atoms[new_atom] = []
new_atoms[new_atom].append(Database.Proof(binding, original_rule))
self.log(atom.table, "new tuples generated: %s", iterstr(new_atoms))
# enqueue each distinct generated tuple, recording appropriate bindings
for new_atom in new_atoms:
# self.log(event.table, "new_tuple %s: %s", new_tuple,
# new_tuples[new_tuple])
# Only enqueue if new data.
# Putting the check here is necessary to support recursion.
self.enqueue(
Event(formula=new_atom,
proofs=new_atoms[new_atom],
insert=insert))
def update_would_cause_errors(self, events):
"""Return a list of PolicyException.
Return a list of PolicyException if we were
to apply the insert/deletes of policy statements dictated by
EVENTS to the current policy.
"""
self.log(None, "update_would_cause_errors %s", utility.iterstr(events))
errors = []
for event in events:
if not compile.is_datalog(event.formula):
errors.append(
exception.PolicyException("Non-formula found: {}".format(
str(event.formula))))
else:
if event.formula.is_atom():
errors.extend(
compile.fact_errors(event.formula, self.theories,
self.name))
else:
errors.extend(
compile.rule_errors(event.formula, self.theories,
self.name))
# Would also check that rules are non-recursive, but that
# is currently being handled by Runtime. The current implementation
# disallows recursion in all theories.
return errors
def update_would_cause_errors(self, events):
"""Return a list of PolicyException.
Return a list of PolicyException if we were
to apply the insert/deletes of policy statements dictated by
EVENTS to the current policy.
"""
self.log(None, "update_would_cause_errors %s", iterstr(events))
errors = []
for event in events:
if not compile.is_datalog(event.formula):
errors.append(PolicyException(
"Non-formula found: {}".format(
str(event.formula))))
else:
if event.formula.is_atom():
errors.extend(compile.fact_errors(
event.formula, self.theories, self.name))
else:
errors.extend(compile.rule_errors(
event.formula, self.theories, self.name))
# Would also check that rules are non-recursive, but that
# is currently being handled by Runtime. The current implementation
# disallows recursion in all theories.
return errors
def update_would_cause_errors(self, events):
"""Return a list of PolicyException.
Return a list of PolicyException if we were
to apply the events EVENTS to the current policy.
"""
self.log(None, "update_would_cause_errors %s", iterstr(events))
errors = []
for event in events:
if not compile.is_datalog(event.formula):
errors.append(PolicyException(
"Non-formula found: {}".format(
str(event.formula))))
else:
if event.formula.is_atom():
errors.extend(compile.fact_errors(
event.formula, self.theories, self.name))
else:
errors.extend(compile.rule_head_has_no_theory(
event.formula,
permit_head=lambda lit: lit.is_update()))
# Should put this back in place, but there are some
# exceptions that we don't handle right now.
# Would like to mark some tables as only being defined
# for certain bound/free arguments and take that into
# account when doing error checking.
# errors.extend(compile.rule_negation_safety(event.formula))
return errors
def process_new_bindings(self, bindings, atom, insert, original_rule):
"""Process new bindings.
For each of BINDINGS, apply to ATOM, and enqueue it as an insert if
INSERT is True and as a delete otherwise.
"""
# for each binding, compute generated tuple and group bindings
# by the tuple they generated
new_atoms = {}
for binding in bindings:
new_atom = atom.plug(binding)
if new_atom not in new_atoms:
new_atoms[new_atom] = []
new_atoms[new_atom].append(database.Database.Proof(
binding, original_rule))
self.log(atom.table.table, "new tuples generated: %s",
utility.iterstr(new_atoms))
# enqueue each distinct generated tuple, recording appropriate bindings
for new_atom in new_atoms:
# self.log(event.table, "new_tuple %s: %s", new_tuple,
# new_tuples[new_tuple])
# Only enqueue if new data.
# Putting the check here is necessary to support recursion.
self.enqueue(compile.Event(formula=new_atom,
proofs=new_atoms[new_atom],
insert=insert))
def update(self, events):
"""Apply EVENTS.
And return the list of EVENTS that actually
changed the theory. Each event is the insert or delete of
a policy statement.
"""
changes = []
self.log(None, "Update %s", utility.iterstr(events))
try:
for event in events:
schema_changes = self.update_rule_schema(
event.formula, event.insert)
formula = compile.reorder_for_safety(event.formula)
if event.insert:
if self._insert_actual(formula):
changes.append(event)
else:
self.revert_schema(schema_changes)
else:
if self._delete_actual(formula):
changes.append(event)
else:
self.revert_schema(schema_changes)
except Exception:
LOG.exception("runtime caught an exception")
raise
return changes
def update_would_cause_errors(self, events):
"""Return a list of PolicyException.
Return a list of PolicyException if we were
to apply the events EVENTS to the current policy.
"""
self.log(None, "update_would_cause_errors %s", utility.iterstr(events))
errors = []
for event in events:
if not compile.is_datalog(event.formula):
errors.append(
exception.PolicyException("Non-formula found: {}".format(
str(event.formula))))
else:
if event.formula.is_atom():
errors.extend(
compile.fact_errors(event.formula, self.theories,
self.name))
else:
errors.extend(
compile.rule_head_has_no_theory(
event.formula,
permit_head=lambda lit: lit.is_update()))
# Should put this back in place, but there are some
# exceptions that we don't handle right now.
# Would like to mark some tables as only being defined
# for certain bound/free arguments and take that into
# account when doing error checking.
# errors.extend(compile.rule_negation_safety(event.formula))
return errors
def modify(self, event):
"""Modifies contents of theory to insert/delete FORMULA.
Returns True iff the theory changed.
"""
self.log(None, "Materialized.modify")
self.enqueue_any(event)
changes = self.process_queue()
self.log(event.formula.tablename(), "modify returns %s", utility.iterstr(changes))
return changes
def modify(self, event):
"""Modifies contents of theory to insert/delete FORMULA.
Returns True iff the theory changed.
"""
self.log(None, "Materialized.modify")
self.enqueue_any(event)
changes = self.process_queue()
self.log(event.formula.tablename(), "modify returns %s",
iterstr(changes))
return changes
def insert(self, rule):
"""Insert a compile.Rule into the theory.
Return True iff the theory changed.
"""
assert compile.is_regular_rule(rule), "DeltaRuleTheory only takes rules"
self.log(rule.tablename(), "Insert: %s", rule)
if rule in self.originals:
self.log(None, utility.iterstr(self.originals))
return False
self.log(rule.tablename(), "Insert 2: %s", rule)
for delta in self.compute_delta_rules([rule]):
self.insert_delta(delta)
self.originals.add(rule)
return True
def modify(self, event):
"""Insert/delete the compile.Rule RULE into the theory.
Return list of changes (either the empty list or
a list including just RULE).
"""
self.log(None, "DeltaRuleTheory.modify %s", event.formula)
self.log(None, "originals: %s", iterstr(self.originals))
if event.insert:
if self.insert(event.formula):
return [event]
else:
if self.delete(event.formula):
return [event]
return []
def modify(self, event):
"""Insert/delete the compile.Rule RULE into the theory.
Return list of changes (either the empty list or
a list including just RULE).
"""
self.log(None, "DeltaRuleTheory.modify %s", event.formula)
self.log(None, "originals: %s", utility.iterstr(self.originals))
if event.insert:
if self.insert(event.formula):
return [event]
else:
if self.delete(event.formula):
return [event]
return []
def prepush_processor(self, data, dataindex, type=None):
"""Called before push.
Takes as input the DATA that the receiver needs and returns
the payload for the message. If this is a regular publication
message, make the payload just the delta; otherwise, make the
payload the entire table.
"""
# This routine basically ignores DATA and sends a delta
# of the self.prior_state and self.state, for the DATAINDEX
# part of the state.
self.log("prepush_processor: dataindex <%s> data: %s", dataindex, data)
# if not a regular publication, just return the original data
if type != 'pub':
self.log("prepush_processor: returned original data")
if type == 'sub':
# Always want to send initialization of []
if data is None:
return []
else:
return data
return data
# grab deltas
to_add = self.state_set_diff(self.state, self.prior_state, dataindex)
to_del = self.state_set_diff(self.prior_state, self.state, dataindex)
self.log("to_add: %s", to_add)
self.log("to_del: %s", to_del)
# create Events
result = []
for row in to_add:
formula = compile.Literal.create_from_table_tuple(dataindex, row)
event = compile.Event(formula=formula, insert=True)
result.append(event)
for row in to_del:
formula = compile.Literal.create_from_table_tuple(dataindex, row)
event = compile.Event(formula=formula, insert=False)
result.append(event)
if len(result) == 0:
# Policy engine expects an empty update to be an init msg
# So if delta is empty, return None, which signals
# the message should not be sent.
result = None
text = "None"
else:
text = utility.iterstr(result)
self.log("prepush_processor for <%s> returning with %s items",
dataindex, text)
return result
def insert(self, rule):
"""Insert a compile.Rule into the theory.
Return True iff the theory changed.
"""
assert compile.is_regular_rule(rule), (
"DeltaRuleTheory only takes rules")
self.log(rule.tablename(), "Insert: %s", rule)
if rule in self.originals:
self.log(None, iterstr(self.originals))
return False
self.log(rule.tablename(), "Insert 2: %s", rule)
for delta in self.compute_delta_rules([rule]):
self.insert_delta(delta)
self.originals.add(rule)
return True
def update_would_cause_errors(self, events):
"""Return a list of Policyxception.
Return a list of PolicyException if we were
to apply the events EVENTS to the current policy.
"""
self.log(None, "update_would_cause_errors %s", utility.iterstr(events))
errors = []
for event in events:
if not compile.is_atom(event.formula):
errors.append(exception.PolicyException(
"Non-atomic formula is not permitted: {}".format(
str(event.formula))))
else:
errors.extend(compile.fact_errors(
event.formula, self.theories, self.name))
return errors
def update_would_cause_errors(self, events):
"""Return a list of PolicyException.
Return a list of PolicyException if we were
to apply the events EVENTS to the current policy.
"""
self.log(None, "update_would_cause_errors %s", utility.iterstr(events))
errors = []
# compute new rule set
for event in events:
assert compile.is_datalog(event.formula), "update_would_cause_errors operates only on objects"
self.log(None, "Updating %s", event.formula)
if event.formula.is_atom():
errors.extend(compile.fact_errors(event.formula, self.theories, self.name))
else:
errors.extend(compile.rule_errors(event.formula, self.theories, self.name))
return errors
def update_would_cause_errors(self, events):
"""Return a list of Policyxception.
Return a list of PolicyException if we were
to apply the events EVENTS to the current policy.
"""
self.log(None, "update_would_cause_errors %s", utility.iterstr(events))
errors = []
for event in events:
if not compile.is_atom(event.formula):
errors.append(
exception.PolicyException(
"Non-atomic formula is not permitted: {}".format(
str(event.formula))))
else:
errors.extend(
compile.fact_errors(event.formula, self.theories,
self.name))
return errors
def update_would_cause_errors(self, events):
"""Return a list of PolicyException.
Return a list of PolicyException if we were
to apply the events EVENTS to the current policy.
"""
self.log(None, "update_would_cause_errors %s", iterstr(events))
errors = []
# compute new rule set
for event in events:
assert compile.is_datalog(event.formula), (
"update_would_cause_errors operates only on objects")
self.log(None, "Updating %s", event.formula)
if event.formula.is_atom():
errors.extend(
compile.fact_errors(event.formula, self.theories,
self.name))
else:
errors.extend(
compile.rule_errors(event.formula, self.theories,
self.name))
return errors
def __str__(self):
return ("TopDownCaller<variables={}, binding={}, find_all={}, "
"results={}, save={}, support={}>".format(
iterstr(self.variables), str(self.binding),
str(self.find_all), iterstr(self.results),
repr(self.save), iterstr(self.support)))
def test_empty(self):
xs = map(TestIterstr.X, range(0))
observed = utility.iterstr(xs)
self.assertEqual("[]", str(observed))
self.assertEqual("[]", repr(observed))
def test_logging_skips_interpolation(self):
with self.get_logging_fixtures() as (stream, handler, logger):
iterable = utility.iterstr(map(TestIterstr.X, range(5)))
logger.debug("some message %s", iterable)
self.assertIsNone(iterable._str_interp)
def test_logging_basic_integration(self):
with self.get_logging_fixtures() as (stream, handler, logger):
iterable = utility.iterstr(map(TestIterstr.X, range(5)))
logger.info("some message %s", iterable)
handler.flush()
self.assertEqual("some message [0;1;2;3;4]\n", stream.getvalue())
def test_empty(self):
xs = map(TestIterstr.X, range(0))
observed = utility.iterstr(xs)
self.assertEqual("[]", str(observed))
self.assertEqual("[]", repr(observed))
def test_logging_basic_integration(self):
with self.get_logging_fixtures() as (stream, handler, logger):
iterable = utility.iterstr(map(TestIterstr.X, range(5)))
logger.info("some message %s", iterable)
handler.flush()
self.assertEqual("some message [0;1;2;3;4]\n", stream.getvalue())
def test__repr__returns_formal_representation(self):
xs = map(TestIterstr.X, range(5))
observed = utility.iterstr(xs)
self.assertEqual("[X:0;X:1;X:2;X:3;X:4]", repr(observed))
self.assertEqual("[X:0;X:1;X:2;X:3;X:4]", "{!r}".format(observed))
self.assertEqual("[X:0;X:1;X:2;X:3;X:4]", "%r" % observed)
def test__str__returns_informal_representation(self):
xs = map(TestIterstr.X, range(5))
observed = utility.iterstr(xs)
self.assertEqual("[0;1;2;3;4]", str(observed))
self.assertEqual("[0;1;2;3;4]", "{}".format(observed))
self.assertEqual("[0;1;2;3;4]", "%s" % observed)
def __str__(self):
return "TopDownResult(binding={}, support={})".format(
unify.binding_str(self.binding), iterstr(self.support))
def __str__(self):
return "TopDownResult(binding={}, support={})".format(
unify.binding_str(self.binding), utility.iterstr(self.support))
def test_logging_skips_interpolation(self):
with self.get_logging_fixtures() as (stream, handler, logger):
iterable = utility.iterstr(map(TestIterstr.X, range(5)))
logger.debug("some message %s", iterable)
self.assertIsNone(iterable._str_interp)
def test__str__returns_informal_representation(self):
xs = map(TestIterstr.X, range(5))
observed = utility.iterstr(xs)
self.assertEqual("[0;1;2;3;4]", str(observed))
self.assertEqual("[0;1;2;3;4]", "{}".format(observed))
self.assertEqual("[0;1;2;3;4]", "%s" % observed)
def test__repr__returns_formal_representation(self):
xs = map(TestIterstr.X, range(5))
observed = utility.iterstr(xs)
self.assertEqual("[X:0;X:1;X:2;X:3;X:4]", repr(observed))
self.assertEqual("[X:0;X:1;X:2;X:3;X:4]", "{!r}".format(observed))
self.assertEqual("[X:0;X:1;X:2;X:3;X:4]", "%r" % observed)
