def loadVariables(self):
"""Loads the instance variables of this object with appropriate
values. This method is useful when the DAG is loaded from a DOT file.
"""
self.initializeDictionaries()
self.resetEdgeWeights()
logger.info("Initializing data structures for special edges...")
self._initSpecialEdges()
logger.info("Data structures initialized.")
def getLatestVersionInfo():
"""Obtains information about the latest available version of GameTime.
Returns:
Dictionary that contains information about the latest available
version of GameTime. If the dictionary is empty, then no such
information could be obtained.
If, however, the dictionary is not empty, it has at least
the two following keys:
- `version`, whose corresponding value is the number of
the latest version of GameTime, and
- `info_url`, whose corresponding value is the URL of
the webpage that provides more information about
the latest version of GameTime.
"""
try:
logger.info("Retrieving data about the latest version of GameTime...")
urlHandler = urllib2.urlopen(config.LATEST_VERSION_INFO_URL, timeout=3)
latestVersionInfo = json.load(urlHandler)
logger.info("Data retrieved.")
if not latestVersionInfo.has_key("version"):
raise ValueError("Data retrieved does not have the number of "
"the latest version of GameTime.")
elif not latestVersionInfo.has_key("info_url"):
raise ValueError("Data retrieved does not have URL of the webpage "
"that provides more information about "
"the latest version of GameTime.")
return latestVersionInfo
except urllib2.URLError as e:
errorReason = e.reason
# Special case
if len(e.args) > 0 and e.args[0].errno == 11001:
errorReason = "No Internet connection detected."
logger.warning("Unable to retrieve data about the latest "
"version of GameTime: %s" % errorReason)
return {}
except socket.timeout as e:
logger.warning("Unable to retrieve data about the latest "
"version of GameTime: The attempt to connect with "
"the URL that provides this data timed out.")
return {}
except ValueError as e:
logger.warning("Data retrieved was in an incorrect format: %s" % e)
return {}
def writeHistogramToFile(location, paths, bins=10, range=None, measured=False):
"""Computes a histogram from the values of a list of
feasible paths generated by GameTime, and writes the histogram
to a file. Each line of the file has the left edge of each bin
and the number of samples in each bin, with both of the values
separated by whitespace.
Arguments:
location:
Location of the file.
paths:
List of feasible paths generated by GameTime, each
represented by a :class:`~gametime.path.Path` object.
bins:
Same purpose as the same-named argument of the function
:func:`numpy.histogram`.
range:
Same purpose as the same-named argument of the function
:func:`numpy.histogram`.
measured:
`True` if, and only if, the values that will be used for
the histogram are the measured values of the feasible paths.
"""
logger.info("Creating histogram...")
hist, binEdges = computeHistogram(paths, bins, range, measured)
try:
histogramFileHandler = open(location, "w")
except EnvironmentError as e:
errMsg = ("Error writing the histogram to the file located "
"at %s: %s" % (location, e))
raise GameTimeError(errMsg)
else:
with histogramFileHandler:
for binEdge, sample in zip(binEdges, hist):
histogramFileHandler.write("%s\t%s\n" % (binEdge, sample))
logger.info("Histogram created.")
def readProjectConfigFile(location):
"""
Reads project configuration information from the XML file provided.
Arguments:
location:
Location of the XML file that contains project
configuration information.
Returns:
:class:`~gametime.projectConfiguration.ProjectConfiguration` object
that contains information from the XML file whose location is provided.
"""
logger.info("Reading project configuration in %s..." % location)
if not os.path.exists(location):
errMsg = "Cannot find project configuration file: %s" % location
raise GameTimeError(errMsg)
try:
projectConfigDom = minidom.parse(location)
except EnvironmentError as e:
errMsg = "Error reading from project configuration file: %s" % e
raise GameTimeError(errMsg)
# Check that the root element is properly named.
rootNode = projectConfigDom.documentElement
if rootNode.tagName != 'gametime-project':
raise GameTimeError("The root element in the XML file should be "
"named `gametime-project'.")
# Check that no child element of the root element has an illegal tag.
rootChildNodes = [
node for node in rootNode.childNodes
if node.nodeType == node.ELEMENT_NODE
]
for childNode in rootChildNodes:
childNodeTag = childNode.tagName
if childNodeTag not in ["file", "preprocess", "analysis", "debug"]:
raise GameTimeError("Unrecognized tag: %s" % childNodeTag)
# Find the directory that contains the project configuration XML file.
projectConfigDir = os.path.dirname(os.path.abspath(location))
# Initialize the instantiation variables for
# the ProjectConfiguration object.
locationFile, func = "", ""
startLabel, endLabel = "", ""
included, merged, inlined, unrollLoops = [], [], [], False
randomizeInitialBasis = False
maximumErrorScaleFactor = 10
determinantThreshold, maxInfeasiblePaths = 0.001, 100
modelAsNestedArrays, preventBasisRefinement = False, False
ilpSolverName, smtSolverName = "", ""
# Process information about the file to be analyzed.
fileNode = (projectConfigDom.getElementsByTagName("file"))[0]
for node in fileNode.childNodes:
if node.nodeType == node.ELEMENT_NODE:
nodeText = _getText(node)
nodeTag = node.tagName
if nodeTag == "location":
locationFile = \
os.path.normpath(os.path.join(projectConfigDir, nodeText))
elif nodeTag == "analysis-function":
func = nodeText
elif nodeTag == "start-label":
startLabel = nodeText
elif nodeTag == "end-label":
endLabel = nodeText
else:
raise GameTimeError("Unrecognized tag: %s" % nodeTag)
# Process the preprocessing variables and flags.
preprocessingNode = \
(projectConfigDom.getElementsByTagName("preprocess"))[0]
for node in preprocessingNode.childNodes:
if node.nodeType == node.ELEMENT_NODE:
nodeText = _getText(node)
nodeTag = node.tagName
if nodeTag == "unroll-loops":
unrollLoops = True
elif nodeTag == "include":
if nodeText != "":
included = getDirPaths(nodeText, projectConfigDir)
elif nodeTag == "merge":
if nodeText != "":
merged = getFilePaths(nodeText, projectConfigDir)
elif nodeTag == "inline":
if nodeText != "":
inlined = getFuncNames(nodeText)
else:
raise GameTimeError("Unrecognized tag: %s" % nodeTag)
# Process the analysis variables and flags.
analysisNode = (projectConfigDom.getElementsByTagName("analysis"))[0]
for node in analysisNode.childNodes:
if node.nodeType == node.ELEMENT_NODE:
nodeText = _getText(node)
nodeTag = node.tagName
if nodeTag == "randomize-initial-basis":
randomizeInitialBasis = True
elif nodeTag == "maximum-error-scale-factor":
maximumErrorScaleFactor = float(nodeText)
elif nodeTag == "determinant-threshold":
determinantThreshold = float(nodeText)
elif nodeTag == "max-infeasible-paths":
maxInfeasiblePaths = int(nodeText)
elif nodeTag == "model-as-nested-arrays":
modelAsNestedArrays = True
elif nodeTag == "prevent-basis-refinement":
preventBasisRefinement = True
elif nodeTag == "ilp-solver":
ilpSolverName = nodeText
elif nodeTag == "smt-solver":
smtSolverName = nodeText
else:
raise GameTimeError("Unrecognized tag: %s" % nodeTag)
# Initialize the instantiation variables for the
# DebugConfiguration object.
keepCilTemps, dumpIr, keepIlpSolverOutput = False, False, False
dumpInstructionTrace, dumpPath, dumpAllPaths = False, False, False
dumpSmtTrace, dumpAllQueries = False, False
keepParserOutput, keepSimulatorOutput = False, False
# Process the debug flags.
debugNode = (projectConfigDom.getElementsByTagName("debug"))[0]
for node in debugNode.childNodes:
if node.nodeType == node.ELEMENT_NODE:
nodeText = _getText(node)
nodeTag = node.tagName
if nodeTag == "keep-cil-temps":
keepCilTemps = True
elif nodeTag == "dump-ir":
dumpIr = True
elif nodeTag == "keep-ilp-solver-output":
keepIlpSolverOutput = True
elif nodeTag == "dump-instruction-trace":
dumpInstructionTrace = True
elif nodeTag == "dump-path":
dumpPath = True
elif nodeTag == "dump-all-paths":
dumpAllPaths = True
elif nodeTag == "dump-smt-trace":
dumpSmtTrace = True
elif nodeTag == "dump-all-queries":
dumpAllQueries = True
elif nodeTag == "keep-parser-output":
keepParserOutput = True
elif nodeTag == "keep-simulator-output":
keepSimulatorOutput = True
else:
raise GameTimeError("Unrecognized tag: %s" % nodeTag)
# Instantiate a DebugConfiguration object.
debugConfig = DebugConfiguration(keepCilTemps, dumpIr, keepIlpSolverOutput,
dumpInstructionTrace, dumpPath,
dumpAllPaths, dumpSmtTrace,
dumpAllQueries, keepParserOutput,
keepSimulatorOutput)
# We have obtained all the information we need from the XML
# file provided. Instantiate a ProjectConfiguration object.
projectConfig = ProjectConfiguration(
locationFile, func, smtSolverName, startLabel, endLabel, included,
merged, inlined, unrollLoops, randomizeInitialBasis,
maximumErrorScaleFactor, determinantThreshold, maxInfeasiblePaths,
modelAsNestedArrays, preventBasisRefinement, ilpSolverName,
debugConfig)
logger.info("Successfully loaded project.")
logger.info("")
return projectConfig
def findExtremePath(analyzer, extremum=Extremum.LONGEST, interval=None):
"""Determines either the longest or the shortest path through the DAG
with the constraints stored in the ``Analyzer`` object provided.
Arguments:
analyzer:
``Analyzer`` object that maintains information about
the code being analyzed.
extremum:
Type of extreme path to calculate.
interval:
``Interval`` object that represents the interval of values
that the generated paths can have. If no ``Interval`` object
is provided, the interval of values is considered to be
all real numbers.
Returns:
Tuple whose first element is the longest or the shortest path
through the DAG, as a list of nodes along the path (ordered
by traversal from source to sink), and whose second element is
the integer linear programming problem that was solved to obtain
the path, as an object of the ``IlpProblem`` class.
If no such path is feasible, given the constraints stored in
the ``Analyzer`` object and the ``Interval`` object provided,
the first element of the tuple is an empty list, and the second
element of the tuple is an ``IlpProblem`` object whose ``objVal``
instance variable is None.
"""
# Make temporary variables for the frequently accessed
# variables from the ``Analyzer`` object provided.
projectConfig = analyzer.projectConfig
dag = analyzer.dag
source = dag.source
sink = dag.sink
numEdges = dag.numEdges
nodesExceptSourceSink = dag.nodesExceptSourceSink
edges = dag.allEdges
edgeWeights = dag.edgeWeights
pathExclusiveConstraints = analyzer.pathExclusiveConstraints
pathBundledConstraints = analyzer.pathBundledConstraints
# Set up the linear programming problem.
logger.info("Setting up the integer linear programming problem...")
problem = IlpProblem(_LP_NAME)
logger.info("Creating the variables and adding the constraints...")
# Set up the variables that correspond to the flow through each edge.
# Set each of the variables to be an integer binary variable.
edgeFlows = pulp.LpVariable.dicts("EdgeFlow", range(0, numEdges), 0, 1,
pulp.LpBinary)
# Add a constraint for the flow from the source. The flow through all of
# the edges out of the source should sum up to exactly 1.
edgeFlowsFromSource = _getEdgeFlowVars(analyzer, edgeFlows,
dag.out_edges(source))
problem += pulp.lpSum(edgeFlowsFromSource) == 1, "Flows from source"
# Add constraints for the rest of the nodes (except sink). The flow
# through all of the edges into a node should equal the flow through
# all of the edges out of the node. Hence, for node n, if e_i and e_j
# enter a node, and e_k and e_l exit a node, the corresponding flow
# equation is e_i + e_j = e_k + e_l.
for node in nodesExceptSourceSink:
edgeFlowsToNode = _getEdgeFlowVars(analyzer, edgeFlows,
dag.in_edges(node))
edgeFlowsFromNode = _getEdgeFlowVars(analyzer, edgeFlows,
dag.out_edges(node))
problem += \
(pulp.lpSum(edgeFlowsToNode) == pulp.lpSum(edgeFlowsFromNode),
"Flows through %s" % node)
# Add a constraint for the flow to the sink. The flow through all of
# the edges into the sink should sum up to exactly 1.
edgeFlowsToSink = _getEdgeFlowVars(analyzer, edgeFlows, dag.in_edges(sink))
problem += pulp.lpSum(edgeFlowsToSink) == 1, "Flows to sink"
# Add constraints for the exclusive path constraints. To ensure that
# the edges in each constraint are not taken together, the total flow
# through all the edges should add to at least one unit less than
# the number of edges in the constraint. Hence, if a constraint
# contains edges e_a, e_b, e_c, then e_a + e_b + e_c must be less than 3.
# This way, all three of these edges can never be taken together.
for constraintNum, path in enumerate(pathExclusiveConstraints):
edgeFlowsInConstraint = _getEdgeFlowVars(analyzer, edgeFlows, path)
problem += (pulp.lpSum(edgeFlowsInConstraint) <= (len(path) - 1),
"Path exclusive constraint %d" % (constraintNum + 1))
# Add constraints for the bundled path constraints. If a constraint
# contains edges e_a, e_b, e_c, e_d, and each edge *must* be taken,
# then e_b + e_c + e_d must sum up to e_a, scaled by -3 (or one less
# than the number of edges in the path constraint). Hence, the flow
# constraint is e_b + e_c + e_d = -3 * e_a. By default, we scale
# the first edge in a constraint with this negative value.
for constraintNum, path in enumerate(pathBundledConstraints):
firstEdge = path[0]
firstEdgeFlow = _getEdgeFlowVar(analyzer, edgeFlows, firstEdge)
edgeFlowsForRest = _getEdgeFlowVars(analyzer, edgeFlows, path[1:])
problem += \
(pulp.lpSum(edgeFlowsForRest) == (len(path)-1) * firstEdgeFlow,
"Path bundled constraint %d" % (constraintNum+1))
# There may be bounds on the values of the paths that are generated
# by this function: we add constraints for these bounds. For this,
# we weight the PuLP variables for the edges using the list of
# edge weights provided, and then impose bounds on the sum.
weightedEdgeFlowVars = []
for edgeIndex, edgeFlowVar in edgeFlows.items():
edgeWeight = edgeWeights[edgeIndex]
weightedEdgeFlowVars.append(edgeWeight * edgeFlowVar)
interval = interval or Interval()
if interval.hasFiniteLowerBound():
problem += \
(pulp.lpSum(weightedEdgeFlowVars) >= interval.lowerBound)
if interval.hasFiniteUpperBound():
problem += \
(pulp.lpSum(weightedEdgeFlowVars) <= interval.upperBound)
logger.info("Variables created and constraints added.")
logger.info("Constructing the objective function...")
# Finally, construct and add the objective function.
# We reuse the constraint (possibly) added in the last step of
# the constraint addition phase.
objective = pulp.lpSum(weightedEdgeFlowVars)
problem += objective
logger.info("Objective function constructed.")
logger.info("Finding the maximum value of the objective function...")
problem.sense = pulp.LpMaximize
problemStatus = problem.solve(solver=projectConfig.ilpSolver)
if problemStatus != pulp.LpStatusOptimal:
logger.info("Maximum value not found.")
return [], problem
objValMax = pulp.value(objective)
logger.info("Maximum value found: %g" % objValMax)
logger.info("Finding the path that corresponds to the maximum value...")
# Determine the edges along the extreme path using the solution.
maxPath = [
edges[edgeNum] for edgeNum in edgeFlows
if edgeFlows[edgeNum].value() == 1
]
logger.info("Path found.")
logger.info("Finding the minimum value of the objective function...")
problem.sense = pulp.LpMinimize
problemStatus = problem.solve(solver=projectConfig.ilpSolver)
if problemStatus != pulp.LpStatusOptimal:
logger.info("Minimum value not found.")
return [], problem
objValMin = pulp.value(objective)
logger.info("Minimum value found: %g" % objValMin)
logger.info("Finding the path that corresponds to the minimum value...")
# Determine the edges along the extreme path using the solution.
minPath = [
edges[edgeNum] for edgeNum in edgeFlows
if edgeFlows[edgeNum].value() == 1
]
logger.info("Path found.")
# Choose the correct extreme path based on the optimal solutions
# and the type of extreme path required.
absMax, absMin = abs(objValMax), abs(objValMin)
if extremum is Extremum.LONGEST:
extremePath = maxPath if absMax >= absMin else minPath
problem.sense = (pulp.LpMaximize
if absMax >= absMin else pulp.LpMinimize)
problem.objVal = max(absMax, absMin)
elif extremum is Extremum.SHORTEST:
extremePath = minPath if absMax >= absMin else maxPath
problem.sense = (pulp.LpMinimize
if absMax >= absMin else pulp.LpMaximize)
problem.objVal = min(absMax, absMin)
# Arrange the nodes along the extreme path in order of traversal
# from source to sink.
resultPath = []
logger.info("Arranging the nodes along the chosen extreme path "
"in order of traversal...")
# To do so, first construct a dictionary mapping a node along the path
# to the edge from that node.
extremePathDict = {}
for edge in extremePath:
extremePathDict[edge[0]] = edge
# Now, "thread" a path through the dictionary.
currNode = source
resultPath.append(currNode)
while currNode in extremePathDict:
newEdge = extremePathDict[currNode]
currNode = newEdge[1]
resultPath.append(currNode)
logger.info("Nodes along the chosen extreme path arranged.")
if projectConfig.debugConfig.KEEP_ILP_SOLVER_OUTPUT:
_moveIlpFiles(os.getcwd(), projectConfig.locationTempDir)
else:
_removeTempIlpFiles()
# We're done!
return resultPath, problem
def findMinimalOvercompleteBasis(analyzer, paths, k):
"""
This function is here only for test purposes. The functions finds the
smallest set of 'basis paths' with the following property: Each path pi
in `paths', can be expressed as a linear combination
pi = a_1 b_1 + ... + a_n b_n
of paths b_i from `basis`. This function finds the set of basis paths
such that every path can be expressed as a linear combination of basis
paths b_i such that the sum of absolute value of coefficients is at
most 'k':
|a_1| + |a_2| + ... + |a_n| <= k
Arguments
analyzer:
``Analyzer`` object that maintains information about
the code being analyzed.
paths:
List of paths that we want to find out how well can be
expressed as a linear combination of paths in `basis`
k:
bound on how well the 'paths' can be expressed as a linear
combination of the calculated basis paths
Returns:
List of paths satisfying the condition stated above
"""
dag = analyzer.dag
source = dag.soure
sink = dag.sink
numEdges = dag.numEdges
edges = dag.edges()
numPaths = len(paths)
# Set up the linear programming problem.
logger.info("Number of paths: %d " % numPaths)
logger.info("Setting up the integer linear programming problem...")
problem = IlpProblem(_LP_NAME)
logger.info("Creating variables")
indices = [(i, j) for i in range(numPaths) for j in range(numPaths)]
coeffs = pulp.LpVariable.dicts("c", indices, -k, k)
absValues = pulp.LpVariable.dicts("abs", indices, 0, k)
usedPaths = pulp.LpVariable.dicts("used", range(numPaths), 0, 1,
pulp.LpBinary)
logger.info("Adding usedPaths")
for i in range(numPaths):
for j in range(numPaths):
problem += k * usedPaths[j] >= absValues[(i, j)]
logger.info("Add absolute values")
for index in indices:
problem += absValues[index] >= coeffs[index]
problem += absValues[index] >= -coeffs[index]
for i in range(numPaths):
logger.info("Processing path number %d" % i)
# all coefficients expressing path i
allCoeffExpressing = [absValues[(i, j)] for j in range(numPaths)]
problem += pulp.lpSum(allCoeffExpressing) <= k
for edge in edges:
pathsContainingEdge = \
[j for j in range(numPaths) if (edge in paths[j])]
presentCoeffs = [coeffs[(i, j)] for j in pathsContainingEdge]
present = 1 if (edge in paths[i]) else 0
problem += pulp.lpSum(presentCoeffs) == present
objective = pulp.lpSum(usedPaths)
problem += objective
problem.sense = pulp.LpMinimize
problemStatus = problem.solve(solver=projectConfig.ilpSolver)
if problemStatus != pulp.LpStatusOptimal:
logger.info("Minimum value not found.")
return [], problem
objValMin = pulp.value(objective)
logger.info("Minimum value found: %g" % objValMin)
solutionPaths = \
[index for index in range(numPaths) if usedPaths[index].value() == 1]
return solutionPaths
def findGoodnessOfFit(analyzer, paths, basis):
"""
This function is here only for test purposes. Each path pi in `paths',
can be expressed as a linear combination
pi = a_1 b_1 + ... + a_n b_n
of paths b_i from `basis`. This function returns the least number `c`
such that every path can be expressed as a linear combination of basis
paths b_i such that the sum of absolute value of coefficients is at
most `c`:
|a_1| + |a_2| + ... + |a_n| <= c
Arguments
analyzer:
``Analyzer`` object that maintains information about
the code being analyzed.
paths:
List of paths that we want to find out how well can be
expressed as a linear combination of paths in `basis`
basis:
List of paths that are used to express `paths` as a linear
combination of
Returns:
The number `c` as described in the paragraph above.
"""
dag = analyzer.dag
source = dag.source
sink = dag.sink
numEdges = dag.numEdges
edges = dag.edges()
numPaths = len(paths)
numBasis = len(basis)
projectConfig = analyzer.projectConfig
# Set up the linear programming problem.
logger.info("Number of paths: %d " % numPaths)
logger.info("Number of basis paths: %d " % numBasis)
logger.info("Setting up the integer linear programming problem...")
problem = IlpProblem("BLAH")
logger.info("Creating variables")
indices = [(i, j) for i in range(numPaths) for j in range(numBasis)]
coeffs = pulp.LpVariable.dicts("c", indices, -100, 100)
absValues = pulp.LpVariable.dicts("abs", indices, 0, 100)
bound = pulp.LpVariable("bnd", 0, 10000)
logger.info("Add absolute values")
for index in indices:
problem += absValues[index] >= coeffs[index]
problem += absValues[index] >= -coeffs[index]
for i in range(numPaths):
# all coefficients expressing path i
allCoeffExpressing = [absValues[(i, j)] for j in range(numBasis)]
problem += pulp.lpSum(allCoeffExpressing) <= bound
# express path i as a linear combination of basis paths
for edge in edges:
pathsContainingEdge = \
[j for j in range(numBasis) if (edge in basis[j])]
presentCoeffs = [coeffs[(i, j)] for j in pathsContainingEdge]
present = 1 if (edge in paths[i]) else 0
problem += pulp.lpSum(presentCoeffs) == present
problem += bound
problem.sense = pulp.LpMinimize
problemStatus = problem.solve(solver=projectConfig.ilpSolver)
if problemStatus != pulp.LpStatusOptimal:
logger.info("Minimum value not found.")
return [], problem
objValMin = pulp.value(bound)
logger.info("Minimum value found: %g" % objValMin)
return objValMin
def generateAndSolveCoreProblem(analyzer,
paths,
pathFunctionUpper,
pathFunctionLower,
weightsPositive,
printProblem=False,
extremum=Extremum.LONGEST):
"""This function actually constructs the ILP to find the longest path
in the graph specified by 'analyzer' using the set of measured paths given
by 'paths'.
Arguments
analyzer:
``Analyzer`` object that maintains information about
the code being analyzed. Among others, contains the underlying
DAG or the collection of infeasible paths.
paths:
List of paths used in the measurements. Each path is a list of
edges in the order in which they are visited by the path
pathFunctionUpper:
Function of type: path -> float that for a given path should
return the upper bound on the length of the given path. The
input 'path' is always from 'paths'
pathFunctionLower:
Function of type: path -> float that for a given path should
return the upper bound on the length of the given path. The
input 'path' is always from 'paths'
weightsPositive:
Boolean value specifying whether the individual edge weight are
required to be at least 0 (if set to True) or can be arbitrary
real value (if set to False)
printProblem:
Boolean value used for debugging. If set to true, the generated
ILP is printed.
extremum:
Specifies whether we are calculating Extremum.LONGEST or
Extremum.SHORTEST
Returns:
Triple consisting of the length of the longest path found, the actual
path and the ILP problem generated.
"""
dag = analyzer.dag
dag.initializeDictionaries()
source = dag.source
sink = dag.sink
numEdges = dag.numEdges
edges = dag.edges()
numPaths = len(paths)
# Use the compact representation of the DAG
# compact is now a mapping that for each edge of dag gives an index of an
# edge in the compact graph.
compact = makeCompact(dag)
projectConfig = analyzer.projectConfig
nodesExceptSourceSink = dag.nodesExceptSourceSink
pathExclusiveConstraints = analyzer.pathExclusiveConstraints
pathBundledConstraints = analyzer.pathBundledConstraints
# Set up the linear programming problem.
logger.info("Number of paths: %d " % numPaths)
logger.info("Setting up the integer linear programming problem...")
problem = IlpProblem(_LP_NAME)
# Take M to be the maximum edge length. Add 1.0 to make sure there are
# no problems due to rounding errors.
M = max([pathFunctionUpper(path) for path in paths] + [0]) + 1.0
if not weightsPositive: M *= numEdges
logger.info("Using value %.2f for M --- the maximum edge weight" % M)
logger.info("Creating variables")
values = set()
for key in compact:
values.add(compact[key])
new_edges = len(values)
# Set up the variables that correspond to the flow through each edge.
# Set each of the variables to be an integer binary variable.
edgeFlows = pulp.LpVariable.dicts("EdgeFlow", range(0, new_edges), 0, 1,
pulp.LpBinary)
edgeWeights = pulp.LpVariable.dicts("we", range(0, new_edges),
0 if weightsPositive else -M, M)
# for a given 'path' in the original DAG returns the edgeFlow variables
# corresponding to the edges along the same path in the compact DAG.
def getNewIndices(compact, edgeFlows, path):
edges = [compact[edge] for edge in path]
pathWeights = [edgeFlows[edge] for edge in set(edges)]
return pathWeights
for path in paths:
pathWeights = \
getNewIndices(compact, edgeWeights, dag.getEdges(path.nodes))
problem += pulp.lpSum(pathWeights) <= pathFunctionUpper(path)
problem += pulp.lpSum(pathWeights) >= pathFunctionLower(path)
# Add a constraint for the flow from the source. The flow through all of
# the edges out of the source should sum up to exactly 1.
edgeFlowsFromSource = \
getNewIndices(compact, edgeFlows, dag.out_edges(source))
problem += pulp.lpSum(edgeFlowsFromSource) == 1, "Flows from source"
# Add constraints for the rest of the nodes (except sink). The flow
# through all of the edges into a node should equal the flow through
# all of the edges out of the node. Hence, for node n, if e_i and e_j
# enter a node, and e_k and e_l exit a node, the corresponding flow
# equation is e_i + e_j = e_k + e_l.
for node in nodesExceptSourceSink:
if (dag.neighbors(node) == 1) and (dag.predecessors(node) == 1):
continue
edgeFlowsToNode = getNewIndices(compact, edgeFlows, dag.in_edges(node))
edgeFlowsFromNode = getNewIndices(compact, edgeFlows,
dag.out_edges(node))
problem += \
(pulp.lpSum(edgeFlowsToNode) == pulp.lpSum(edgeFlowsFromNode),
"Flows through %s" % node)
# Add a constraint for the flow to the sink. The flow through all of
# the edges into the sink should sum up to exactly 1.
edgeFlowsToSink = getNewIndices(compact, edgeFlows, dag.in_edges(sink))
problem += pulp.lpSum(edgeFlowsToSink) == 1, "Flows to sink"
# Add constraints for the exclusive path constraints. To ensure that
# the edges in each constraint are not taken together, the total flow
# through all the edges should add to at least one unit less than
# the number of edges in the constraint. Hence, if a constraint
# contains edges e_a, e_b, e_c, then e_a + e_b + e_c must be less than 3.
# This way, all three of these edges can never be taken together.
for constraintNum, path in enumerate(pathExclusiveConstraints):
edgeFlowsInConstraint = getNewIndices(compact, edgeFlows, path)
problem += (pulp.lpSum(edgeFlowsInConstraint) <=
(len(edgeFlowsInConstraint) - 1),
"Path exclusive constraint %d" % (constraintNum + 1))
# Each productVars[index] in the longest path should correspond to
# edgeFlows[index] * edgeWeights[index]
productVars = pulp.LpVariable.dicts("pe", range(0, new_edges), -M, M)
for index in range(0, new_edges):
if extremum == Extremum.LONGEST:
problem += productVars[index] <= edgeWeights[index]
problem += productVars[index] <= M * edgeFlows[index]
else:
problem += productVars[index] >= edgeWeights[index] - M * (
1.0 - edgeFlows[index])
problem += productVars[index] >= 0
objective = pulp.lpSum(productVars)
problem += objective
logger.info("Objective function constructed.")
if extremum == Extremum.LONGEST:
logger.info("Finding the maximum value of the objective function...")
problem.sense = pulp.LpMaximize
else:
logger.info("Finding the minimum value of the objective function...")
problem.sense = pulp.LpMinimize
problemStatus = problem.solve(solver=projectConfig.ilpSolver)
if (printProblem): logger.info(problem)
if problemStatus != pulp.LpStatusOptimal:
logger.info("Maximum value not found.")
return -1, [], problem
objValMax = pulp.value(objective)
problem.objVal = objValMax
logger.info("Maximum value found: %g" % objValMax)
logger.info("Finding the path that corresponds to the maximum value...")
# Determine the edges along the extreme path using the solution.
maxPath = [
edges[edgeNum] for edgeNum in edgeFlows
if edgeFlows[edgeNum].value() > 0.1
]
logger.info("Path found.")
totalLength = sum([
productVars[edgeNum].value() for edgeNum in edgeFlows
if edgeFlows[edgeNum].value() == 1
])
logger.info("Total length of the path %.2f" % totalLength)
objValMax = totalLength
maxPath = [
edgeNum for edgeNum in range(0, new_edges)
if edgeFlows[edgeNum].value() > 0.1
]
extremePath = []
#reverse exremePath according to the compact edgeMap
for edge in maxPath:
map_to = [source for source in compact if compact[source] == edge]
extremePath.extend(map_to)
# Arrange the nodes along the extreme path in order of traversal
# from source to sink.
resultPath = []
logger.info("Arranging the nodes along the chosen extreme path "
"in order of traversal...")
# To do so, first construct a dictionary mapping a node along the path
# to the edge from that node.
extremePathDict = {}
for edge in extremePath:
extremePathDict[edge[0]] = edge
# Now, "thread" a path through the dictionary.
currNode = dag.source
resultPath.append(currNode)
while currNode in extremePathDict:
newEdge = extremePathDict[currNode]
currNode = newEdge[1]
resultPath.append(currNode)
logger.info("Nodes along the chosen extreme path arranged.")
if projectConfig.debugConfig.KEEP_ILP_SOLVER_OUTPUT:
_moveIlpFiles(os.getcwd(), projectConfig.locationTempDir)
else:
_removeTempIlpFiles()
# We're done!
return objValMax, resultPath, problem
def findLeastCompatibleMuMax(analyzer, paths):
"""This function returns the least dealta in the underlying graph, as
specified by 'analyzer', that is feasible with the given set of
measurements as specified by 'paths'. The method does not take into
account which paths are feasible and which not; it considers all the
paths in the graph.
Arguments:
analyzer:
``Analyzer`` object that maintains information about
the code being analyzed.
paths:
List of paths used in the measurements. Each path is a list of
edges in the order in which they are visited by the path
Returns:
A floting point value---the least delta compatible with the
measurements
"""
dag = analyzer.dag
source = dag.source
sink = dag.sink
numEdges = dag.numEdges
edges = dag.edges()
numPaths = len(paths)
projectConfig = analyzer.projectConfig
nodesExceptSourceSink = dag.nodesExceptSourceSink
# Set up the linear programming problem.
logger.info("Number of paths: %d " % numPaths)
logger.info("Setting up the integer linear programming problem...")
problem = IlpProblem(_LP_NAME)
logger.info("Creating variables")
# Set up the variables that correspond to weights of each edge.
# Each edge is restricted to be a nonnegative real number
edgeWeights = pulp.LpVariable.dicts("we", range(0, numEdges), 0)
# Create the variable that shall correspond to the least delta
delta = pulp.LpVariable("delta", 0)
for path in paths:
pathWeights = \
_getEdgeFlowVars(analyzer, edgeWeights, dag.getEdges(path.nodes))
problem += pulp.lpSum(pathWeights) <= delta + path.measuredValue
problem += pulp.lpSum(pathWeights) >= -delta + path.measuredValue
print "LENGTH:", path.measuredValue
# Optimize for the least delta
problem += delta
logger.info("Finding the minimum value of the objective function...")
problem.sense = pulp.LpMinimize
problemStatus = problem.solve(solver=projectConfig.ilpSolver)
if problemStatus != pulp.LpStatusOptimal:
logger.info("Maximum value not found.")
return []
objValMin = pulp.value(delta)
logger.info("Minimum compatible delta found: %g" % objValMin)
if projectConfig.debugConfig.KEEP_ILP_SOLVER_OUTPUT:
_moveIlpFiles(os.getcwd(), projectConfig.locationTempDir)
else:
_removeTempIlpFiles()
return objValMin
def generatePaths(analyzer,
numPaths=5,
pathType=PathType.WORST_CASE,
interval=None,
useObExtraction=False):
"""Generates a list of feasible paths of the code being analyzed,
each represented by an object of the ``Path`` class.
The type of the generated paths is determined by the pathType
argument, which is a class variable of the ``PathType`` class.
By default, this argument is ``PathType.WORST_CASE``. For
a description of the types, refer to the documentation of
the ``PathType`` class.
The ``numPaths`` argument is an upper bound on how many paths should
be generated, which is 5 by default. This argument is ignored if
this method is used to generate all of the feasible paths of the code
being analyzed.
The ``interval`` argument is an ``Interval`` object that represents
the interval of values that the generated paths can have. If
no ``Interval`` object is provided, the interval of values is
considered to be all real numbers.
This method is idempotent: a second call to this method will produce
the same list of ``Path`` objects as the first call, assuming that
nothing has changed between the two calls.
Precondition: The basis ``Path`` objects of the input ``Analyzer``
object have values associated with them. Refer to either the method
``loadBasisValuesFromFile`` or the method ``loadBasisValues`` in
the ``Analyzer`` class.
Arguments:
analyzer:
``Analyzer`` object that maintains information about
the code being analyzed.
numPaths:
Upper bound on the number of paths to generate.
pathType:
Type of paths to generate, represented by a class variable of
the ``PathType`` class. The different types of paths are
described in the documentation of the ``PathType`` class.
interval:
``Interval`` object that represents the interval of
values that the generated paths can have. If no
``Interval`` object is provided, the interval of values
is considered to be all real numbers.
useObExtraction:
Boolean value specifiying whether to use overcomplete
basis extraction algorithm
Returns:
List of feasible paths of the code being analyzed, each
represented by an object of the ``Path`` class.
"""
paths = None
if pathType == PathType.WORST_CASE:
logger.info("Generating %d worst-case feasible paths..." %
numPaths)
paths = \
PathGenerator._generatePaths(analyzer, numPaths,
pulpHelper.Extremum.LONGEST,
interval, useObExtraction)
elif pathType == PathType.BEST_CASE:
logger.info("Generating %d best-case feasible paths..." % numPaths)
paths = \
PathGenerator._generatePaths(analyzer, numPaths,
pulpHelper.Extremum.SHORTEST,
interval, useObExtraction)
if paths is not None:
logger.info("%d of %d paths have been generated." %
(len(paths), numPaths))
return paths
if pathType == PathType.ALL_DECREASING:
logger.info("Generating all feasible paths in decreasing order "
"of value...")
paths = \
PathGenerator._generatePaths(analyzer, analyzer.dag.numPaths,
pulpHelper.Extremum.LONGEST,
interval, useObExtraction)
elif pathType == PathType.ALL_INCREASING:
logger.info("Generating all feasible paths in increasing order "
"of value...")
paths = \
PathGenerator._generatePaths(analyzer, analyzer.dag.numPaths,
pulpHelper.Extremum.SHORTEST,
interval, useObExtraction)
if paths is not None:
logger.info("%d feasible paths have been generated." % len(paths))
return paths
if pathType == PathType.RANDOM:
logger.info("Generating random feasible paths...")
paths = \
PathGenerator._generatePaths(analyzer, numPaths, None, interval)
logger.info("%d of %d paths have been generated." %
(len(paths), numPaths))
return paths
else:
raise GameTimeError("Unrecognized path type: %d" % pathType)
def _generatePaths(analyzer,
numPaths,
extremum=pulpHelper.Extremum.LONGEST,
interval=None,
useObExtraction=False):
"""Helper static method for the ``generatePaths`` static method.
Generates a list of feasible paths of the code being analyzed,
each represented by an object of the ``Path`` class.
Arguments:
analyzer:
``Analyzer`` object that maintains information about
the code being analyzed.
numPaths:
Upper bound on the number of paths to generate.
extremum:
Type of paths to calculate (longest or shortest),
represented by an element of the ``Extremum`` class in
the ``pulpHelper`` module.
interval:
``Interval`` object that represents the interval of
values that the generated paths can have. If
no ``Interval`` object is provided, the interval of values
is considered to be all real numbers.
useObExtraction:
Boolean value specifiying whether to use overcomplete
basis extraction algorithm
Returns:
List of feasible paths of the code being analyzed,
each represented by an object of the ``Path`` class.
"""
if nxHelper.hasCycles(analyzer.dag):
logger.warn("Loops in the code have been detected.")
logger.warn("No feasible paths have been generated.")
return []
logger.info("")
startTime = time.clock()
if useObExtraction:
beforeTime = time.clock()
logger.info("Using the new algorithm to extract the longest path")
logger.info("Finding Least Compatible Delta")
muMax = pulpHelper.findLeastCompatibleMuMax(
analyzer, analyzer.basisPaths)
logger.info("Found the least mu_max compatible with measurements: "
"%.2f in %.2f seconds" %
(muMax, time.clock() - beforeTime))
analyzer.inferredMuMax = muMax
beforeTime = time.clock()
logger.info("Calculating error bounds in the estimate")
analyzer.errorScaleFactor, path, ilpProblem = \
pulpHelper.findWorstExpressiblePath(
analyzer, analyzer.basisPaths, 0)
logger.info(
"Total maximal error in estimates is 2 x %.2f x %.2f = %.2f" %
(analyzer.errorScaleFactor, muMax,
2 * analyzer.errorScaleFactor * muMax))
logger.info("Calculated in %.2f ms" % (time.clock() - beforeTime))
else:
analyzer.estimateEdgeWeights()
resultPaths = []
currentPathNum, numPathsUnsat, numCandidatePaths = 0, 0, 0
while (currentPathNum < numPaths
and numCandidatePaths < analyzer.dag.numPaths):
logger.info("Currently generating path %d..." %
(currentPathNum + 1))
logger.info("So far, %d candidate paths were found to be "
"unsatisfiable." % numPathsUnsat)
if analyzer.pathDimension == 1:
warnMsg = ("Basis matrix has dimensions 1x1. "
"There is only one path through the function "
"under analysis.")
logger.warn(warnMsg)
logger.info("Finding a candidate path using an integer "
"linear program...")
logger.info("")
if extremum is None:
source, sink = analyzer.dag.source, analyzer.dag.sink
candidatePathNodes = nxHelper.getRandomPath(
analyzer.dag, source, sink)
candidatePathEdges = Dag.getEdges(candidatePathNodes)
analyzer.addPathBundledConstraint(candidatePathEdges)
if useObExtraction:
candidatePathNodes, ilpProblem = \
pulpHelper.findLongestPathWithDelta(
analyzer, analyzer.basisPaths, muMax, extremum)
else:
candidatePathNodes, ilpProblem = \
pulpHelper.findExtremePath(analyzer,
extremum if extremum is not None
else pulpHelper.Extremum.LONGEST,
interval)
logger.info("")
if ilpProblem.objVal is None:
if (extremum is not None
or numCandidatePaths == analyzer.dag.numPaths):
logger.info("Unable to find a new candidate path.")
break
elif extremum is None:
analyzer.addPathExclusiveConstraint(candidatePathEdges)
analyzer.resetPathBundledConstraints()
numCandidatePaths = len(analyzer.pathExclusiveConstraints)
continue
logger.info("Candidate path found.")
candidatePathEdges = Dag.getEdges(candidatePathNodes)
candidatePathValue = ilpProblem.objVal
logger.info("Checking if candidate path is feasible...")
logger.info("")
resultPath = analyzer.checkFeasibility(candidatePathNodes,
ilpProblem)
querySatisfiability = resultPath.smtQuery.satisfiability
if querySatisfiability == Satisfiability.SAT:
logger.info("Candidate path is feasible.")
resultPath.setPredictedValue(candidatePathValue)
resultPaths.append(resultPath)
logger.info("Path %d generated." % (currentPathNum + 1))
# Exclude the path generated from future iterations.
analyzer.addPathExclusiveConstraint(candidatePathEdges)
currentPathNum += 1
numPathsUnsat = 0
elif querySatisfiability == Satisfiability.UNSAT:
logger.info("Candidate path is infeasible.")
logger.info("Finding the edges to exclude...")
unsatCore = resultPath.smtQuery.unsatCore
excludeEdges = resultPath.getEdgesForConditions(unsatCore)
logger.info("Edges to be excluded found.")
logger.info("Adding constraint to exclude these edges...")
if len(excludeEdges) > 0:
analyzer.addPathExclusiveConstraint(excludeEdges)
else:
analyzer.addPathExclusiveConstraint(candidatePathEdges)
logger.info("Constraint added.")
numPathsUnsat += 1
numCandidatePaths += 1
if extremum is None:
analyzer.resetPathBundledConstraints()
logger.info("")
logger.info("")
analyzer.resetPathExclusiveConstraints()
logger.info("Time taken to generate paths: %.2f seconds." %
(time.clock() - startTime))
return resultPaths