def test_merge(self):
c = metrics.MapCounter("foo")
c.inc("x")
c.inc("y", 2)
# Cheat a little by merging a counter with a different name.
other = metrics.MapCounter("other")
other.inc("x")
other.inc("z")
c._merge(other)
# Check merged contents.
self.assertEqual(5, c._total)
self.assertDictEqual(dict(x=2, y=2, z=1), c._counts)
def test_enabled(self):
c = metrics.MapCounter("foo")
# Check contents of an empty map.
self.assertEqual(0, c._total)
# Increment a few values and check again.
c.inc("x")
c.inc("y", 2)
c.inc("x", 5)
self.assertEqual(8, c._total)
self.assertDictEqual(dict(x=6, y=2), c._counts)
self.assertEqual("foo: 8 {x=6, y=2}", str(c))
def test_merge_from(self):
# Create a counter, increment it, and dump it.
c1 = metrics.Counter("foo")
c1.inc(1)
dump = metrics.dump([c1], encoding=None)
# Reset metrics, merge from dump, which will create a new metric.
metrics._prepare_for_test()
self.assertFalse(metrics._registered_metrics)
metrics.merge_from_file(moves.cStringIO(dump))
m = metrics._registered_metrics["foo"]
self.assertEqual(1, m._total)
# Merge again, this time it will merge data into the existing metric.
metrics.merge_from_file(moves.cStringIO(dump))
self.assertEqual(2, m._total)
# It's an error to merge an incompatible type.
metrics._prepare_for_test()
_ = metrics.MapCounter("foo")
self.assertRaises(TypeError, metrics.merge_from_file,
moves.cStringIO(dump))
"""
def __init__(self, node, dnf):
# The condition is represented by a dummy variable with a single binding
# to None. The origins for this binding are the dnf clauses.
self._var = node.program.NewVariable()
self._binding = self._var.AddBinding(None)
for clause in dnf:
sources = set(clause)
self._binding.AddOrigin(node, sources)
@property
def binding(self):
return self._binding
_restrict_counter = metrics.MapCounter("state_restrict")
def split_conditions(node, var):
"""Return a pair of conditions for the value being true and false."""
return (_restrict_condition(node, var.bindings, True),
_restrict_condition(node, var.bindings, False))
def _restrict_condition(node, bindings, logical_value):
"""Return a restricted condition based on filtered bindings.
Args:
node: The CFGNode.
bindings: A sequence of bindings.
logical_value: Either True or False.
def test_disabled(self):
metrics._prepare_for_test(enabled=False)
c = metrics.MapCounter("foo")
c.inc("x")
self.assertEqual(0, c._total)
class Solver(object):
"""The solver class is instantiated for a given "problem" instance.
It maintains a cache of solutions for subproblems to be able to recall them if
they reoccur in the solving process.
"""
_cache_metric = metrics.MapCounter("cfg_solver_cache")
_goals_per_find_metric = metrics.Distribution("cfg_solver_goals_per_find")
def __init__(self, program):
"""Initialize a solver instance. Every instance has their own cache.
Arguments:
program: The program we're in.
"""
self.program = program
self._solved_states = {}
self._path_finder = _PathFinder()
def Solve(self, start_attrs, start_node):
"""Try to solve the given problem.
Try to prove one or more bindings starting (and going backwards from) a
given node, all the way to the program entrypoint.
Arguments:
start_attrs: The assignments we're trying to have, at the start node.
start_node: The CFG node where we want the assignments to be active.
Returns:
True if there is a path through the program that would give "start_attr"
its binding at the "start_node" program position. For larger programs,
this might only look for a partial path (i.e., a path that doesn't go
back all the way to the entry point of the program).
"""
state = State(start_node, start_attrs)
return self._RecallOrFindSolution(state)
def _RecallOrFindSolution(self, state):
"""Memoized version of FindSolution()."""
if state in self._solved_states:
Solver._cache_metric.inc("hit")
return self._solved_states[state]
# To prevent infinite loops, we insert this state into the hashmap as a
# solvable state, even though we have not solved it yet. The reasoning is
# that if it's possible to solve this state at this level of the tree, it
# can also be solved in any of the children.
self._solved_states[state] = True
Solver._cache_metric.inc("miss")
result = self._solved_states[state] = self._FindSolution(state)
return result
def _FindSolution(self, state):
"""Find a sequence of assignments that would solve the given state."""
if state.pos.condition:
state.goals.add(state.pos.condition)
Solver._goals_per_find_metric.add(len(state.goals))
for removed_goals, new_goals in state.RemoveFinishedGoals():
assert not state.pos.bindings & new_goals
if _GoalsConflict(removed_goals):
continue # We bulk-removed goals that are internally conflicting.
if not new_goals:
return True
blocked = frozenset().union(*(goal.variable.nodes
for goal in new_goals))
new_positions = set()
for goal in new_goals:
# "goal" is the assignment we're trying to find.
for origin in goal.origins:
path_exist, path = self._path_finder.FindNodeBackwards(
state.pos, origin.where, blocked)
if path_exist:
where = origin.where
# Check if we found conditions on the way.
for node in path:
if node is not state.pos:
where = node
break
new_positions.add(where)
for new_pos in new_positions:
new_state = State(new_pos, new_goals)
if self._RecallOrFindSolution(new_state):
return True
return False
class Solver(object):
"""The solver class is instantiated for a given "problem" instance.
It maintains a cache of solutions for subproblems to be able to recall them if
they reoccur in the solving process.
"""
_cache_metric = metrics.MapCounter("cfg_solver_cache")
_goals_per_find_metric = metrics.Distribution("cfg_solver_goals_per_find")
def __init__(self, program):
"""Initialize a solver instance. Every instance has their own cache.
Arguments:
program: The program we're in.
"""
self.program = program
self._solved_states = {}
self._path_finder = _PathFinder()
def Solve(self, start_attrs, start_node):
"""Try to solve the given problem.
Try to prove one or more bindings starting (and going backwards from) a
given node, all the way to the program entrypoint.
Arguments:
start_attrs: The assignments we're trying to have, at the start node.
start_node: The CFG node where we want the assignments to be active.
Returns:
True if there is a path through the program that would give "start_attr"
its binding at the "start_node" program position. For larger programs,
this might only look for a partial path (i.e., a path that doesn't go
back all the way to the entry point of the program).
"""
state = State(start_node, start_attrs)
return self._RecallOrFindSolution(state, frozenset(start_attrs))
def _RecallOrFindSolution(self, state, seen_goals):
"""Memoized version of FindSolution()."""
if state in self._solved_states:
Solver._cache_metric.inc("hit")
return self._solved_states[state]
# To prevent infinite loops, we insert this state into the hashmap as a
# solvable state, even though we have not solved it yet. The reasoning is
# that if it's possible to solve this state at this level of the tree, it
# can also be solved in any of the children.
self._solved_states[state] = True
Solver._cache_metric.inc("miss")
result = self._solved_states[state] = self._FindSolution(
state, seen_goals)
return result
def _FindSolution(self, state, seen_goals):
"""Find a sequence of assignments that would solve the given state."""
if state.Done():
return True
if _GoalsConflict(state.goals):
return False
Solver._goals_per_find_metric.add(len(state.goals))
# Note that this set might contain the current CFG node:
blocked = frozenset(state.NodesWithAssignments())
# Find the goal cfg node that was assigned last. Due to the fact that we
# treat CFGs as DAGs, there's typically one unique cfg node with this
# property.
for goal in state.goals:
# "goal" is the assignment we're trying to find.
for origin in goal.origins:
path_exist, path = self._path_finder.FindNodeBackwards(
state.pos, origin.where, blocked)
if path_exist:
# This loop over multiple different combinations of origins is why
# we need memoization of states.
for source_set in origin.source_sets:
new_goals = set(state.goals)
where = origin.where
# If we found conditions on the way, see whether we need to add
# any of them to our goals.
for node in path:
if node.condition not in seen_goals:
# It can happen that node == state.pos, typically if the node
# we're calling HasCombination on has a condition. If so, we'll
# treat it like any other condition and add it to our goals.
new_goals.add(node.condition)
where = node
break
new_state = State(where, new_goals)
if origin.where is new_state.pos:
# The goal can only be replaced if origin.where was actually
# reached.
new_state.Replace(goal, source_set)
# Also remove all goals that are trivially fulfilled at the
# new CFG node.
removed = new_state.RemoveFinishedGoals()
removed.add(goal)
if _GoalsConflict(removed | new_state.goals):
pass # We bulk-removed goals that are internally conflicting.
elif self._RecallOrFindSolution(
new_state, seen_goals | new_goals):
return True
return False
class Solver(object):
"""The solver class is instantiated for a given "problem" instance.
It maintains a cache of solutions for subproblems to be able to recall them if
they reoccur in the solving process.
"""
_cache_metric = metrics.MapCounter("cfg_solver_cache")
_goals_per_find_metric = metrics.Distribution("cfg_solver_goals_per_find")
def __init__(self, program):
"""Initialize a solver instance. Every instance has their own cache.
Arguments:
program: The program we're in.
"""
self.program = program
self._solved_states = {}
def Solve(self, start_attrs, start_node):
"""Try to solve the given problem.
Try to prove one or more bindings starting (and going backwards from) a
given node, all the way to the program entrypoint.
Arguments:
start_attrs: The assignments we're trying to have, at the start node.
start_node: The CFG node where we want the assignments to be active.
Returns:
True if there is a path through the program that would give "start_attr"
its binding at the "start_node" program position. For larger programs,
this might only look for a partial path (i.e., a path that doesn't go
back all the way to the entry point of the program).
"""
state = State(start_node, start_attrs)
return self._RecallOrFindSolution(state)
def _RecallOrFindSolution(self, state):
"""Memoized version of FindSolution()."""
if state in self._solved_states:
Solver._cache_metric.inc("hit")
return self._solved_states[state]
# To prevent infinite loops, we insert this state into the hashmap as a
# solvable state, even though we have not solved it yet. The reasoning is
# that if it's possible to solve this state at this level of the tree, it
# can also be solved in any of the children.
self._solved_states[state] = True
Solver._cache_metric.inc("miss")
result = self._solved_states[state] = self._FindSolution(state)
return result
def _FindSolution(self, state):
"""Find a sequence of assignments that would solve the given state."""
if state.Done():
return True
if state.HasConflictingGoals():
return False
Solver._goals_per_find_metric.add(len(state.goals))
blocked = state.NodesWithAssignments()
# We don't treat our current CFG node as blocked: If one of the goal
# variables is overwritten by an assignment at our current pos, we assume
# that assignment can still see the previous bindings.
blocked.discard(state.pos)
blocked = frozenset(blocked)
# Find the goal cfg node that was assigned last. Due to the fact that we
# treat CFGs as DAGs, there's typically one unique cfg node with this
# property.
for goal in state.goals:
# "goal" is the assignment we're trying to find.
for origin in goal.origins:
if _FindNodeBackwards(state.pos, origin.where, blocked):
# This loop over multiple different combinations of origins is why
# we need memoization of states.
for source_set in origin.source_sets:
new_state = State(origin.where, state.goals)
new_state.Replace(goal, source_set)
# Also remove all goals that are trivially fulfilled at the
# new CFG node.
new_state.RemoveFinishedGoals()
if self._RecallOrFindSolution(new_state):
return True
return False
class Solver(object):
"""The solver class is instantiated for a given "problem" instance.
It maintains a cache of solutions for subproblems to be able to recall them if
they reoccur in the solving process.
"""
_cache_metric = metrics.MapCounter("cfg_solver_cache")
_goals_per_find_metric = metrics.Distribution("cfg_solver_goals_per_find")
def __init__(self, program):
"""Initialize a solver instance. Every instance has their own cache.
Arguments:
program: The program we're in.
"""
self.program = program
self._solved_states = {}
self._path_finder = _PathFinder()
def Solve(self, start_attrs, start_node):
"""Try to solve the given problem.
Try to prove one or more bindings starting (and going backwards from) a
given node, all the way to the program entrypoint.
Arguments:
start_attrs: The assignments we're trying to have, at the start node.
start_node: The CFG node where we want the assignments to be active.
Returns:
True if there is a path through the program that would give "start_attr"
its binding at the "start_node" program position. For larger programs,
this might only look for a partial path (i.e., a path that doesn't go
back all the way to the entry point of the program).
"""
state = State(start_node, start_attrs)
return self._RecallOrFindSolution(state)
def _RecallOrFindSolution(self, state):
"""Memoized version of FindSolution()."""
if state in self._solved_states:
Solver._cache_metric.inc("hit")
return self._solved_states[state]
# To prevent infinite loops, we insert this state into the hashmap as a
# solvable state, even though we have not solved it yet. The reasoning is
# that if it's possible to solve this state at this level of the tree, it
# can also be solved in any of the children.
self._solved_states[state] = True
Solver._cache_metric.inc("miss")
result = self._solved_states[state] = self._FindSolution(state)
return result
def _IsSolvedBefore(self, where, goal, entrypoint, blocked):
"""Determine if a goal is possibly solved in subsection of the CFG.
If a condition introduces a new goal, but we can solve that goal *before*
the goal we were trying to solve originally, assume that goal doesn't
have anything to do with us.
This currently does a quick CFG check as an approximation. An alternative
implementation would be to call _FindSolution while blocking the new
entrypoint.
Args:
where: Current CFG node. We search backwards from this node.
goal: The goal to find a solution for.
entrypoint: The "new" entry point of the graph. This typically reduces
the CFG to a subgraph.
blocked: A list of nodes.
Returns:
True if we think this goal can be solved without traversing beyond
"entrypoint", False if it can't.
"""
blocked = frozenset(blocked | {entrypoint})
for origin in goal.origins:
# TODO(kramm): We don't cache this. Should we?
if origin.where not in blocked and self._path_finder.FindPathToNode(
where, origin.where, blocked):
return True
return False
def _FindSolution(self, state):
"""Find a sequence of assignments that would solve the given state."""
if state.Done():
return True
if _GoalsConflict(state.goals):
return False
Solver._goals_per_find_metric.add(len(state.goals))
blocked = state.NodesWithAssignments()
# We don't treat our current CFG node as blocked: If one of the goal
# variables is overwritten by an assignment at our current pos, we assume
# that assignment can still see the previous bindings.
blocked.discard(state.pos)
blocked = frozenset(blocked)
# Find the goal cfg node that was assigned last. Due to the fact that we
# treat CFGs as DAGs, there's typically one unique cfg node with this
# property.
for goal in state.goals:
# "goal" is the assignment we're trying to find.
for origin in goal.origins:
path_exist, path = self._path_finder.FindNodeBackwards(
state.pos, origin.where, blocked)
if path_exist:
# This loop over multiple different combinations of origins is why
# we need memoization of states.
for source_set in origin.source_sets:
new_goals = set(state.goals)
where = origin.where
# If we found conditions on the way, see whether we need to add
# any of them to our goals.
for node in path:
if node.condition not in state.goals and not self._IsSolvedBefore(
node, node.condition, origin.where,
blocked):
# TODO(kramm): what if node == state.pos?
new_goals.add(node.condition)
where = node
break
new_state = State(where, new_goals)
if origin.where is new_state.pos:
# The goal can only be replaced if origin.where was actually
# reached.
new_state.Replace(goal, source_set)
# Also remove all goals that are trivially fulfilled at the
# new CFG node.
removed = new_state.RemoveFinishedGoals()
removed.add(goal)
if _GoalsConflict(removed):
# Sometimes, we bulk-remove goals that are internally conflicting.
return False
if self._RecallOrFindSolution(new_state):
return True
return False