def process_queue(self): """Data and rule propagation routine. Returns list of events that were not noops """ self.log(None, "Processing queue") history = [] while len(self.queue) > 0: event = self.queue.dequeue() self.log(event.tablename(), "Dequeued %s", event) if compile.is_regular_rule(event.formula): changes = self.delta_rules.modify(event) if len(changes) > 0: history.extend(changes) bindings = self.top_down_evaluation( event.formula.variables(), event.formula.body) self.log(event.formula.tablename(), "new bindings after top-down: %s", iterstr(bindings)) self.process_new_bindings(bindings, event.formula.head, event.insert, event.formula) else: self.propagate(event) history.extend(self.database.modify(event)) self.log(event.tablename(), "History: %s", iterstr(history)) return history
def initialize_tables(self, tablenames, formulas, target=None): """Event handler for (re)initializing a collection of tables.""" # translate FORMULAS into list of formula objects actual_formulas = [] formula_tables = set() if isinstance(formulas, basestring): formulas = self.parse(formulas) for formula in formulas: if isinstance(formula, basestring): formula = self.parse1(formula) elif isinstance(formula, tuple): formula = compile.Literal.create_from_iter(formula) assert formula.is_atom() actual_formulas.append(formula) formula_tables.add(formula.table) tablenames = set(tablenames) | formula_tables self.table_log(None, "Initializing tables %s with %s", iterstr(tablenames), iterstr(actual_formulas)) # implement initialization by computing the requisite # update. theory = self.get_target(target) old = set(theory.content(tablenames=tablenames)) new = set(actual_formulas) to_add = new - old to_rem = old - new to_add = [Event(formula_, insert=True) for formula_ in to_add] to_rem = [Event(formula_, insert=False) for formula_ in to_rem] self.table_log(None, "Initialize converted to update with %s and %s", iterstr(to_add), iterstr(to_rem)) return self.update(to_add + to_rem, target=target)
def simulate_obj(self, query, theory, sequence, action_theory, delta, trace): """Simulate objects. Both THEORY and ACTION_THEORY are names of theories. Both QUERY and SEQUENCE are parsed. """ assert compile.is_datalog(query), "Query must be formula" # Each action is represented as a rule with the actual action # in the head and its supporting data (e.g. options) in the body assert all(compile.is_extended_datalog(x) for x in sequence), ( "Sequence must be an iterable of Rules") th_object = self.get_target(theory) if trace: old_tracer = self.get_tracer() tracer = StringTracer() # still LOG.debugs trace tracer.trace('*') # trace everything self.set_tracer(tracer) # if computing delta, query the current state if delta: self.table_log(query.tablename(), "** Simulate: Querying %s", query) oldresult = th_object.select(query) self.table_log(query.tablename(), "Original result of %s is %s", query, iterstr(oldresult)) # apply SEQUENCE self.table_log(query.tablename(), "** Simulate: Applying sequence %s", iterstr(sequence)) undo = self.project(sequence, theory, action_theory) # query the resulting state self.table_log(query.tablename(), "** Simulate: Querying %s", query) result = th_object.select(query) self.table_log(query.tablename(), "Result of %s is %s", query, iterstr(result)) # rollback the changes self.table_log(query.tablename(), "** Simulate: Rolling back") self.project(undo, theory, action_theory) # if computing the delta, do it if delta: result = set(result) oldresult = set(oldresult) pos = result - oldresult neg = oldresult - result pos = [formula.make_update(is_insert=True) for formula in pos] neg = [formula.make_update(is_insert=False) for formula in neg] result = pos + neg if trace: self.set_tracer(old_tracer) return (result, tracer.get_value()) return result
def update_would_cause_errors(self, events): """Return a list of compile.CongressException. Return a list of compile.CongressException if we were to apply the events EVENTS to the current policy. """ self.log(None, "update_would_cause_errors %s", iterstr(events)) errors = [] for event in events: if not compile.is_datalog(event.formula): errors.append( compile.CongressException("Non-formula found: {}".format( str(event.formula)))) else: if event.formula.is_atom(): errors.extend( compile.fact_errors(event.formula, self.theories, self.name)) else: errors.extend( compile.rule_head_has_no_theory( event.formula, permit_head=lambda lit: lit.is_update())) # Should put this back in place, but there are some # exceptions that we don't handle right now. # Would like to mark some tables as only being defined # for certain bound/free arguments and take that into # account when doing error checking. # errors.extend(compile.rule_negation_safety(event.formula)) return errors
def update_would_cause_errors(self, events): """Return a list of compile.CongressException. Return a list of compile.CongressException if we were to apply the insert/deletes of policy statements dictated by EVENTS to the current policy. """ self.log(None, "update_would_cause_errors %s", iterstr(events)) errors = [] for event in events: if not compile.is_datalog(event.formula): errors.append( compile.CongressException("Non-formula found: {}".format( str(event.formula)))) else: if event.formula.is_atom(): errors.extend( compile.fact_errors(event.formula, self.theories, self.name)) else: errors.extend( compile.rule_errors(event.formula, self.theories, self.name)) # Would also check that rules are non-recursive, but that # is currently being handled by Runtime. The current implementation # disallows recursion in all theories. return errors
def process_new_bindings(self, bindings, atom, insert, original_rule): """Process new bindings. For each of BINDINGS, apply to ATOM, and enqueue it as an insert if INSERT is True and as a delete otherwise. """ # for each binding, compute generated tuple and group bindings # by the tuple they generated new_atoms = {} for binding in bindings: new_atom = atom.plug(binding) if new_atom not in new_atoms: new_atoms[new_atom] = [] new_atoms[new_atom].append(Database.Proof(binding, original_rule)) self.log(atom.table, "new tuples generated: %s", iterstr(new_atoms)) # enqueue each distinct generated tuple, recording appropriate bindings for new_atom in new_atoms: # self.log(event.table, "new_tuple %s: %s", new_tuple, # new_tuples[new_tuple]) # Only enqueue if new data. # Putting the check here is necessary to support recursion. self.enqueue( Event(formula=new_atom, proofs=new_atoms[new_atom], insert=insert))
def update_obj(self, events): """Do the updating. Checks if applying EVENTS is permitted and if not returns a list of errors. If it is permitted, it applies it and then returns a list of changes. In both cases, the return is a 2-tuple (if-permitted, list). """ self.table_log(None, "Updating with %s", iterstr(events)) by_theory = self.group_events_by_target(events) # check that the updates would not cause an error errors = [] actual_events = [] for th, th_events in by_theory.items(): th_obj = self.get_target(th) errors.extend(th_obj.update_would_cause_errors(th_events)) actual_events.extend(th_obj.actual_events(th_events)) # update dependency graph (and undo it if errors) changes = self.global_dependency_graph.formula_update(events) if changes: if self.global_dependency_graph.has_cycle(): # TODO(thinrichs): include path errors.append(compile.CongressException( "Rules are recursive")) self.global_dependency_graph.undo_changes(changes) if len(errors) > 0: return (False, errors) # actually apply the updates changes = [] for th, th_events in by_theory.items(): changes.extend(self.get_target(th).update(events)) return (True, changes)
def modify(self, event): """Modifies contents of theory to insert/delete FORMULA. Returns True iff the theory changed. """ self.log(None, "Materialized.modify") self.enqueue_any(event) changes = self.process_queue() self.log(event.formula.tablename(), "modify returns %s", iterstr(changes)) return changes
def modify(self, event): """Insert/delete the compile.Rule RULE into the theory. Return list of changes (either the empty list or a list including just RULE). """ self.log(None, "DeltaRuleTheory.modify %s", event.formula) self.log(None, "originals: %s", iterstr(self.originals)) if event.insert: if self.insert(event.formula): return [event] else: if self.delete(event.formula): return [event] return []
def insert(self, rule): """Insert a compile.Rule into the theory. Return True iff the theory changed. """ assert compile.is_regular_rule(rule), ( "DeltaRuleTheory only takes rules") self.log(rule.tablename(), "Insert: %s", rule) if rule in self.originals: self.log(None, iterstr(self.originals)) return False self.log(rule.tablename(), "Insert 2: %s", rule) for delta in self.compute_delta_rules([rule]): self.insert_delta(delta) self.originals.add(rule) return True
def execute_obj(self, actions): """Executes the list of ACTION instances one at a time. For now, our execution is just logging. """ LOG.debug("Executing: %s", iterstr(actions)) assert all(compile.is_atom(action) and action.is_ground() for action in actions) action_names = self.get_action_names() assert all(action.table in action_names for action in actions) for action in actions: if not action.is_ground(): if self.logger is not None: self.logger.warn("Unground action to execute: %s", action) continue if self.logger is not None: self.logger.info("%s", action)
def update_would_cause_errors(self, events): """Return a list of compile.CongressException. Return a list of compile.CongressException if we were to apply the events EVENTS to the current policy. """ self.log(None, "update_would_cause_errors %s", iterstr(events)) errors = [] for event in events: if not compile.is_atom(event.formula): errors.append(compile.CongressException( "Non-atomic formula is not permitted: {}".format( str(event.formula)))) else: errors.extend(compile.fact_errors( event.formula, self.theories, self.name)) return errors
def remediate_obj(self, formula): """Find a collection of action invocations That if executed result in FORMULA becoming false. """ actionth = self.theory[self.ACTION_THEORY] classifyth = self.theory[self.CLASSIFY_THEORY] # look at FORMULA if compile.is_atom(formula): pass # TODO(tim): clean up unused variable # output = formula elif compile.is_regular_rule(formula): pass # TODO(tim): clean up unused variable # output = formula.head else: assert False, "Must be a formula" # grab a single proof of FORMULA in terms of the base tables base_tables = classifyth.base_tables() proofs = classifyth.explain(formula, base_tables, False) if proofs is None: # FORMULA already false; nothing to be done return [] # Extract base table literals that make that proof true. # For remediation, we assume it suffices to make any of those false. # (Leaves of proof may not be literals or may not be written in # terms of base tables, despite us asking for base tables-- # because of negation.) leaves = [leaf for leaf in proofs[0].leaves() if (compile.is_atom(leaf) and leaf.table in base_tables)] self.table_log(None, "Leaves: %s", iterstr(leaves)) # Query action theory for abductions of negated base tables actions = self.get_action_names() results = [] for lit in leaves: goal = lit.make_positive() if lit.is_negated(): goal.table = goal.table + "+" else: goal.table = goal.table + "-" # return is a list of goal :- act1, act2, ... # This is more informative than query :- act1, act2, ... for abduction in actionth.abduce(goal, actions, False): results.append(abduction) return results
def compute_route(self, events, theory): """Compute rerouting. When a formula is inserted/deleted (in OPERATION) into a THEORY, it may need to be rerouted to another theory. This function computes that rerouting. Returns a Theory object. """ self.table_log(None, "Computing route for theory %s and events %s", theory.name, iterstr(events)) # Since Enforcement includes Classify and Classify includes Database, # any operation on data needs to be funneled into Enforcement. # Enforcement pushes it down to the others and then # reacts to the results. That is, we really have one big theory # Enforcement + Classify + Database as far as the data is concerned # but formulas can be inserted/deleted into each policy individually. if all([compile.is_atom(event.formula) for event in events]): if (theory is self.theory[self.CLASSIFY_THEORY] or theory is self.theory[self.DATABASE]): return self.theory[self.ENFORCEMENT_THEORY] return theory
def update_would_cause_errors(self, events): """Return a list of compile.CongressException. Return a list of compile.CongressException if we were to apply the events EVENTS to the current policy. """ self.log(None, "update_would_cause_errors %s", iterstr(events)) errors = [] # compute new rule set for event in events: assert compile.is_datalog(event.formula), ( "update_would_cause_errors operates only on objects") self.log(None, "Updating %s", event.formula) if event.formula.is_atom(): errors.extend( compile.fact_errors(event.formula, self.theories, self.name)) else: errors.extend( compile.rule_errors(event.formula, self.theories, self.name)) return errors
def update(self, events): """Apply EVENTS. And return the list of EVENTS that actually changed the theory. Each event is the insert or delete of a policy statement. """ changes = [] self.log(None, "Update %s", iterstr(events)) try: for event in events: formula = compile.reorder_for_safety(event.formula) if event.insert: if self.insert_actual(formula): changes.append(event) else: if self.delete_actual(formula): changes.append(event) except Exception as e: LOG.exception("runtime caught an exception") raise e return changes
def __str__(self): return ("TopDownCaller<variables={}, binding={}, find_all={}, " "results={}, save={}, support={}>".format( iterstr(self.variables), str(self.binding), str(self.find_all), iterstr(self.results), repr(self.save), iterstr(self.support)))
def project(self, sequence, policy_theory, action_theory): """Apply the list of updates SEQUENCE. Apply the list of updates SEQUENCE, where actions are described in ACTION_THEORY. Return an update sequence that will undo the projection. SEQUENCE can include atom insert/deletes, rule insert/deletes, and action invocations. Projecting an action only simulates that action's invocation using the action's description; the results are therefore only an approximation of executing actions directly. Elements of SEQUENCE are just formulas applied to the given THEORY. They are NOT Event()s. SEQUENCE is really a program in a mini-programming language--enabling results of one action to be passed to another. Hence, even ignoring actions, this functionality cannot be achieved by simply inserting/deleting. """ actth = self.theory[action_theory] policyth = self.theory[policy_theory] # apply changes to the state newth = NonrecursiveRuleTheory(abbr="Temp") newth.tracer.trace('*') actth.includes.append(newth) # TODO(thinrichs): turn 'includes' into an object that guarantees # there are no cycles through inclusion. Otherwise we get # infinite loops if actth is not policyth: actth.includes.append(policyth) actions = self.get_action_names(action_theory) self.table_log(None, "Actions: %s", iterstr(actions)) undos = [] # a list of updates that will undo SEQUENCE self.table_log(None, "Project: %s", sequence) last_results = [] for formula in sequence: self.table_log(None, "** Updating with %s", formula) self.table_log(None, "Actions: %s", iterstr(actions)) self.table_log(None, "Last_results: %s", iterstr(last_results)) tablename = formula.tablename() if tablename not in actions: if not formula.is_update(): raise compile.CongressException( "Sequence contained non-action, non-update: " + str(formula)) updates = [formula] else: self.table_log(tablename, "Projecting %s", formula) # define extension of current Actions theory if formula.is_atom(): assert formula.is_ground(), ( "Projection atomic updates must be ground") assert not formula.is_negated(), ( "Projection atomic updates must be positive") newth.define([formula]) else: # instantiate action using prior results newth.define(last_results) self.table_log(tablename, "newth (with prior results) %s", iterstr(newth.content())) bindings = actth.top_down_evaluation( formula.variables(), formula.body, find_all=False) if len(bindings) == 0: continue grounds = formula.plug_heads(bindings[0]) grounds = [act for act in grounds if act.is_ground()] assert all(not lit.is_negated() for lit in grounds) newth.define(grounds) self.table_log(tablename, "newth contents (after action insertion): %s", iterstr(newth.content())) # self.table_log(tablename, "action contents: %s", # iterstr(actth.content())) # self.table_log(tablename, "action.includes[1] contents: %s", # iterstr(actth.includes[1].content())) # self.table_log(tablename, "newth contents: %s", # iterstr(newth.content())) # compute updates caused by action updates = actth.consequences(compile.is_update) updates = self.resolve_conflicts(updates) updates = unify.skolemize(updates) self.table_log(tablename, "Computed updates: %s", iterstr(updates)) # compute results for next time for update in updates: newth.insert(update) last_results = actth.consequences(compile.is_result) last_results = set([atom for atom in last_results if atom.is_ground()]) # apply updates for update in updates: undo = self.project_updates(update, policy_theory) if undo is not None: undos.append(undo) undos.reverse() if actth is not policyth: actth.includes.remove(policyth) actth.includes.remove(newth) return undos
def __str__(self): return "TopDownResult(binding={}, support={})".format( unify.binding_str(self.binding), iterstr(self.support))