def process_queue(self):
"""Data and rule propagation routine.
Returns list of events that were not noops
"""
self.log(None, "Processing queue")
history = []
while len(self.queue) > 0:
event = self.queue.dequeue()
self.log(event.tablename(), "Dequeued %s", event)
if compile.is_regular_rule(event.formula):
changes = self.delta_rules.modify(event)
if len(changes) > 0:
history.extend(changes)
bindings = self.top_down_evaluation(
event.formula.variables(), event.formula.body)
self.log(event.formula.tablename(),
"new bindings after top-down: %s",
iterstr(bindings))
self.process_new_bindings(bindings, event.formula.head,
event.insert, event.formula)
else:
self.propagate(event)
history.extend(self.database.modify(event))
self.log(event.tablename(), "History: %s", iterstr(history))
return history
def initialize_tables(self, tablenames, formulas, target=None):
"""Event handler for (re)initializing a collection of tables."""
# translate FORMULAS into list of formula objects
actual_formulas = []
formula_tables = set()
if isinstance(formulas, basestring):
formulas = self.parse(formulas)
for formula in formulas:
if isinstance(formula, basestring):
formula = self.parse1(formula)
elif isinstance(formula, tuple):
formula = compile.Literal.create_from_iter(formula)
assert formula.is_atom()
actual_formulas.append(formula)
formula_tables.add(formula.table)
tablenames = set(tablenames) | formula_tables
self.table_log(None, "Initializing tables %s with %s",
iterstr(tablenames), iterstr(actual_formulas))
# implement initialization by computing the requisite
# update.
theory = self.get_target(target)
old = set(theory.content(tablenames=tablenames))
new = set(actual_formulas)
to_add = new - old
to_rem = old - new
to_add = [Event(formula_, insert=True) for formula_ in to_add]
to_rem = [Event(formula_, insert=False) for formula_ in to_rem]
self.table_log(None, "Initialize converted to update with %s and %s",
iterstr(to_add), iterstr(to_rem))
return self.update(to_add + to_rem, target=target)
def simulate_obj(self, query, theory, sequence, action_theory, delta,
trace):
"""Simulate objects.
Both THEORY and ACTION_THEORY are names of theories.
Both QUERY and SEQUENCE are parsed.
"""
assert compile.is_datalog(query), "Query must be formula"
# Each action is represented as a rule with the actual action
# in the head and its supporting data (e.g. options) in the body
assert all(compile.is_extended_datalog(x) for x in sequence), (
"Sequence must be an iterable of Rules")
th_object = self.get_target(theory)
if trace:
old_tracer = self.get_tracer()
tracer = StringTracer() # still LOG.debugs trace
tracer.trace('*') # trace everything
self.set_tracer(tracer)
# if computing delta, query the current state
if delta:
self.table_log(query.tablename(),
"** Simulate: Querying %s", query)
oldresult = th_object.select(query)
self.table_log(query.tablename(),
"Original result of %s is %s",
query, iterstr(oldresult))
# apply SEQUENCE
self.table_log(query.tablename(), "** Simulate: Applying sequence %s",
iterstr(sequence))
undo = self.project(sequence, theory, action_theory)
# query the resulting state
self.table_log(query.tablename(), "** Simulate: Querying %s", query)
result = th_object.select(query)
self.table_log(query.tablename(), "Result of %s is %s", query,
iterstr(result))
# rollback the changes
self.table_log(query.tablename(), "** Simulate: Rolling back")
self.project(undo, theory, action_theory)
# if computing the delta, do it
if delta:
result = set(result)
oldresult = set(oldresult)
pos = result - oldresult
neg = oldresult - result
pos = [formula.make_update(is_insert=True) for formula in pos]
neg = [formula.make_update(is_insert=False) for formula in neg]
result = pos + neg
if trace:
self.set_tracer(old_tracer)
return (result, tracer.get_value())
return result
def update_would_cause_errors(self, events):
"""Return a list of compile.CongressException.
Return a list of compile.CongressException if we were
to apply the events EVENTS to the current policy.
"""
self.log(None, "update_would_cause_errors %s", iterstr(events))
errors = []
for event in events:
if not compile.is_datalog(event.formula):
errors.append(
compile.CongressException("Non-formula found: {}".format(
str(event.formula))))
else:
if event.formula.is_atom():
errors.extend(
compile.fact_errors(event.formula, self.theories,
self.name))
else:
errors.extend(
compile.rule_head_has_no_theory(
event.formula,
permit_head=lambda lit: lit.is_update()))
# Should put this back in place, but there are some
# exceptions that we don't handle right now.
# Would like to mark some tables as only being defined
# for certain bound/free arguments and take that into
# account when doing error checking.
# errors.extend(compile.rule_negation_safety(event.formula))
return errors
def update_would_cause_errors(self, events):
"""Return a list of compile.CongressException.
Return a list of compile.CongressException if we were
to apply the insert/deletes of policy statements dictated by
EVENTS to the current policy.
"""
self.log(None, "update_would_cause_errors %s", iterstr(events))
errors = []
for event in events:
if not compile.is_datalog(event.formula):
errors.append(
compile.CongressException("Non-formula found: {}".format(
str(event.formula))))
else:
if event.formula.is_atom():
errors.extend(
compile.fact_errors(event.formula, self.theories,
self.name))
else:
errors.extend(
compile.rule_errors(event.formula, self.theories,
self.name))
# Would also check that rules are non-recursive, but that
# is currently being handled by Runtime. The current implementation
# disallows recursion in all theories.
return errors
def process_new_bindings(self, bindings, atom, insert, original_rule):
"""Process new bindings.
For each of BINDINGS, apply to ATOM, and enqueue it as an insert if
INSERT is True and as a delete otherwise.
"""
# for each binding, compute generated tuple and group bindings
# by the tuple they generated
new_atoms = {}
for binding in bindings:
new_atom = atom.plug(binding)
if new_atom not in new_atoms:
new_atoms[new_atom] = []
new_atoms[new_atom].append(Database.Proof(binding, original_rule))
self.log(atom.table, "new tuples generated: %s", iterstr(new_atoms))
# enqueue each distinct generated tuple, recording appropriate bindings
for new_atom in new_atoms:
# self.log(event.table, "new_tuple %s: %s", new_tuple,
# new_tuples[new_tuple])
# Only enqueue if new data.
# Putting the check here is necessary to support recursion.
self.enqueue(
Event(formula=new_atom,
proofs=new_atoms[new_atom],
insert=insert))
def update_obj(self, events):
"""Do the updating.
Checks if applying EVENTS is permitted and if not
returns a list of errors. If it is permitted, it
applies it and then returns a list of changes.
In both cases, the return is a 2-tuple (if-permitted, list).
"""
self.table_log(None, "Updating with %s", iterstr(events))
by_theory = self.group_events_by_target(events)
# check that the updates would not cause an error
errors = []
actual_events = []
for th, th_events in by_theory.items():
th_obj = self.get_target(th)
errors.extend(th_obj.update_would_cause_errors(th_events))
actual_events.extend(th_obj.actual_events(th_events))
# update dependency graph (and undo it if errors)
changes = self.global_dependency_graph.formula_update(events)
if changes:
if self.global_dependency_graph.has_cycle():
# TODO(thinrichs): include path
errors.append(compile.CongressException(
"Rules are recursive"))
self.global_dependency_graph.undo_changes(changes)
if len(errors) > 0:
return (False, errors)
# actually apply the updates
changes = []
for th, th_events in by_theory.items():
changes.extend(self.get_target(th).update(events))
return (True, changes)
def modify(self, event):
"""Modifies contents of theory to insert/delete FORMULA.
Returns True iff the theory changed.
"""
self.log(None, "Materialized.modify")
self.enqueue_any(event)
changes = self.process_queue()
self.log(event.formula.tablename(), "modify returns %s",
iterstr(changes))
return changes
def modify(self, event):
"""Insert/delete the compile.Rule RULE into the theory.
Return list of changes (either the empty list or
a list including just RULE).
"""
self.log(None, "DeltaRuleTheory.modify %s", event.formula)
self.log(None, "originals: %s", iterstr(self.originals))
if event.insert:
if self.insert(event.formula):
return [event]
else:
if self.delete(event.formula):
return [event]
return []
def insert(self, rule):
"""Insert a compile.Rule into the theory.
Return True iff the theory changed.
"""
assert compile.is_regular_rule(rule), (
"DeltaRuleTheory only takes rules")
self.log(rule.tablename(), "Insert: %s", rule)
if rule in self.originals:
self.log(None, iterstr(self.originals))
return False
self.log(rule.tablename(), "Insert 2: %s", rule)
for delta in self.compute_delta_rules([rule]):
self.insert_delta(delta)
self.originals.add(rule)
return True
def execute_obj(self, actions):
"""Executes the list of ACTION instances one at a time.
For now, our execution is just logging.
"""
LOG.debug("Executing: %s", iterstr(actions))
assert all(compile.is_atom(action) and action.is_ground()
for action in actions)
action_names = self.get_action_names()
assert all(action.table in action_names for action in actions)
for action in actions:
if not action.is_ground():
if self.logger is not None:
self.logger.warn("Unground action to execute: %s", action)
continue
if self.logger is not None:
self.logger.info("%s", action)
def update_would_cause_errors(self, events):
"""Return a list of compile.CongressException.
Return a list of compile.CongressException if we were
to apply the events EVENTS to the current policy.
"""
self.log(None, "update_would_cause_errors %s", iterstr(events))
errors = []
for event in events:
if not compile.is_atom(event.formula):
errors.append(compile.CongressException(
"Non-atomic formula is not permitted: {}".format(
str(event.formula))))
else:
errors.extend(compile.fact_errors(
event.formula, self.theories, self.name))
return errors
def remediate_obj(self, formula):
"""Find a collection of action invocations
That if executed result in FORMULA becoming false.
"""
actionth = self.theory[self.ACTION_THEORY]
classifyth = self.theory[self.CLASSIFY_THEORY]
# look at FORMULA
if compile.is_atom(formula):
pass # TODO(tim): clean up unused variable
# output = formula
elif compile.is_regular_rule(formula):
pass # TODO(tim): clean up unused variable
# output = formula.head
else:
assert False, "Must be a formula"
# grab a single proof of FORMULA in terms of the base tables
base_tables = classifyth.base_tables()
proofs = classifyth.explain(formula, base_tables, False)
if proofs is None: # FORMULA already false; nothing to be done
return []
# Extract base table literals that make that proof true.
# For remediation, we assume it suffices to make any of those false.
# (Leaves of proof may not be literals or may not be written in
# terms of base tables, despite us asking for base tables--
# because of negation.)
leaves = [leaf for leaf in proofs[0].leaves()
if (compile.is_atom(leaf) and
leaf.table in base_tables)]
self.table_log(None, "Leaves: %s", iterstr(leaves))
# Query action theory for abductions of negated base tables
actions = self.get_action_names()
results = []
for lit in leaves:
goal = lit.make_positive()
if lit.is_negated():
goal.table = goal.table + "+"
else:
goal.table = goal.table + "-"
# return is a list of goal :- act1, act2, ...
# This is more informative than query :- act1, act2, ...
for abduction in actionth.abduce(goal, actions, False):
results.append(abduction)
return results
def compute_route(self, events, theory):
"""Compute rerouting.
When a formula is inserted/deleted (in OPERATION) into a THEORY,
it may need to be rerouted to another theory. This function
computes that rerouting. Returns a Theory object.
"""
self.table_log(None, "Computing route for theory %s and events %s",
theory.name, iterstr(events))
# Since Enforcement includes Classify and Classify includes Database,
# any operation on data needs to be funneled into Enforcement.
# Enforcement pushes it down to the others and then
# reacts to the results. That is, we really have one big theory
# Enforcement + Classify + Database as far as the data is concerned
# but formulas can be inserted/deleted into each policy individually.
if all([compile.is_atom(event.formula) for event in events]):
if (theory is self.theory[self.CLASSIFY_THEORY] or
theory is self.theory[self.DATABASE]):
return self.theory[self.ENFORCEMENT_THEORY]
return theory
def update_would_cause_errors(self, events):
"""Return a list of compile.CongressException.
Return a list of compile.CongressException if we were
to apply the events EVENTS to the current policy.
"""
self.log(None, "update_would_cause_errors %s", iterstr(events))
errors = []
# compute new rule set
for event in events:
assert compile.is_datalog(event.formula), (
"update_would_cause_errors operates only on objects")
self.log(None, "Updating %s", event.formula)
if event.formula.is_atom():
errors.extend(
compile.fact_errors(event.formula, self.theories,
self.name))
else:
errors.extend(
compile.rule_errors(event.formula, self.theories,
self.name))
return errors
def update(self, events):
"""Apply EVENTS.
And return the list of EVENTS that actually
changed the theory. Each event is the insert or delete of
a policy statement.
"""
changes = []
self.log(None, "Update %s", iterstr(events))
try:
for event in events:
formula = compile.reorder_for_safety(event.formula)
if event.insert:
if self.insert_actual(formula):
changes.append(event)
else:
if self.delete_actual(formula):
changes.append(event)
except Exception as e:
LOG.exception("runtime caught an exception")
raise e
return changes
def __str__(self):
return ("TopDownCaller<variables={}, binding={}, find_all={}, "
"results={}, save={}, support={}>".format(
iterstr(self.variables), str(self.binding),
str(self.find_all), iterstr(self.results),
repr(self.save), iterstr(self.support)))
def project(self, sequence, policy_theory, action_theory):
"""Apply the list of updates SEQUENCE.
Apply the list of updates SEQUENCE, where actions are described
in ACTION_THEORY. Return an update sequence that will undo the
projection.
SEQUENCE can include atom insert/deletes, rule insert/deletes,
and action invocations. Projecting an action only
simulates that action's invocation using the action's description;
the results are therefore only an approximation of executing
actions directly. Elements of SEQUENCE are just formulas
applied to the given THEORY. They are NOT Event()s.
SEQUENCE is really a program in a mini-programming
language--enabling results of one action to be passed to another.
Hence, even ignoring actions, this functionality cannot be achieved
by simply inserting/deleting.
"""
actth = self.theory[action_theory]
policyth = self.theory[policy_theory]
# apply changes to the state
newth = NonrecursiveRuleTheory(abbr="Temp")
newth.tracer.trace('*')
actth.includes.append(newth)
# TODO(thinrichs): turn 'includes' into an object that guarantees
# there are no cycles through inclusion. Otherwise we get
# infinite loops
if actth is not policyth:
actth.includes.append(policyth)
actions = self.get_action_names(action_theory)
self.table_log(None, "Actions: %s", iterstr(actions))
undos = [] # a list of updates that will undo SEQUENCE
self.table_log(None, "Project: %s", sequence)
last_results = []
for formula in sequence:
self.table_log(None, "** Updating with %s", formula)
self.table_log(None, "Actions: %s", iterstr(actions))
self.table_log(None, "Last_results: %s", iterstr(last_results))
tablename = formula.tablename()
if tablename not in actions:
if not formula.is_update():
raise compile.CongressException(
"Sequence contained non-action, non-update: " +
str(formula))
updates = [formula]
else:
self.table_log(tablename, "Projecting %s", formula)
# define extension of current Actions theory
if formula.is_atom():
assert formula.is_ground(), (
"Projection atomic updates must be ground")
assert not formula.is_negated(), (
"Projection atomic updates must be positive")
newth.define([formula])
else:
# instantiate action using prior results
newth.define(last_results)
self.table_log(tablename, "newth (with prior results) %s",
iterstr(newth.content()))
bindings = actth.top_down_evaluation(
formula.variables(), formula.body, find_all=False)
if len(bindings) == 0:
continue
grounds = formula.plug_heads(bindings[0])
grounds = [act for act in grounds if act.is_ground()]
assert all(not lit.is_negated() for lit in grounds)
newth.define(grounds)
self.table_log(tablename,
"newth contents (after action insertion): %s",
iterstr(newth.content()))
# self.table_log(tablename, "action contents: %s",
# iterstr(actth.content()))
# self.table_log(tablename, "action.includes[1] contents: %s",
# iterstr(actth.includes[1].content()))
# self.table_log(tablename, "newth contents: %s",
# iterstr(newth.content()))
# compute updates caused by action
updates = actth.consequences(compile.is_update)
updates = self.resolve_conflicts(updates)
updates = unify.skolemize(updates)
self.table_log(tablename, "Computed updates: %s",
iterstr(updates))
# compute results for next time
for update in updates:
newth.insert(update)
last_results = actth.consequences(compile.is_result)
last_results = set([atom for atom in last_results
if atom.is_ground()])
# apply updates
for update in updates:
undo = self.project_updates(update, policy_theory)
if undo is not None:
undos.append(undo)
undos.reverse()
if actth is not policyth:
actth.includes.remove(policyth)
actth.includes.remove(newth)
return undos
def __str__(self):
return "TopDownResult(binding={}, support={})".format(
unify.binding_str(self.binding), iterstr(self.support))