def loadVariables(self): """Loads the instance variables of this object with appropriate values. This method is useful when the DAG is loaded from a DOT file. """ self.initializeDictionaries() self.resetEdgeWeights() logger.info("Initializing data structures for special edges...") self._initSpecialEdges() logger.info("Data structures initialized.")
def getLatestVersionInfo(): """Obtains information about the latest available version of GameTime. Returns: Dictionary that contains information about the latest available version of GameTime. If the dictionary is empty, then no such information could be obtained. If, however, the dictionary is not empty, it has at least the two following keys: - `version`, whose corresponding value is the number of the latest version of GameTime, and - `info_url`, whose corresponding value is the URL of the webpage that provides more information about the latest version of GameTime. """ try: logger.info("Retrieving data about the latest version of GameTime...") urlHandler = urllib2.urlopen(config.LATEST_VERSION_INFO_URL, timeout=3) latestVersionInfo = json.load(urlHandler) logger.info("Data retrieved.") if not latestVersionInfo.has_key("version"): raise ValueError("Data retrieved does not have the number of " "the latest version of GameTime.") elif not latestVersionInfo.has_key("info_url"): raise ValueError("Data retrieved does not have URL of the webpage " "that provides more information about " "the latest version of GameTime.") return latestVersionInfo except urllib2.URLError as e: errorReason = e.reason # Special case if len(e.args) > 0 and e.args[0].errno == 11001: errorReason = "No Internet connection detected." logger.warning("Unable to retrieve data about the latest " "version of GameTime: %s" % errorReason) return {} except socket.timeout as e: logger.warning("Unable to retrieve data about the latest " "version of GameTime: The attempt to connect with " "the URL that provides this data timed out.") return {} except ValueError as e: logger.warning("Data retrieved was in an incorrect format: %s" % e) return {}
def writeHistogramToFile(location, paths, bins=10, range=None, measured=False): """Computes a histogram from the values of a list of feasible paths generated by GameTime, and writes the histogram to a file. Each line of the file has the left edge of each bin and the number of samples in each bin, with both of the values separated by whitespace. Arguments: location: Location of the file. paths: List of feasible paths generated by GameTime, each represented by a :class:`~gametime.path.Path` object. bins: Same purpose as the same-named argument of the function :func:`numpy.histogram`. range: Same purpose as the same-named argument of the function :func:`numpy.histogram`. measured: `True` if, and only if, the values that will be used for the histogram are the measured values of the feasible paths. """ logger.info("Creating histogram...") hist, binEdges = computeHistogram(paths, bins, range, measured) try: histogramFileHandler = open(location, "w") except EnvironmentError as e: errMsg = ("Error writing the histogram to the file located " "at %s: %s" % (location, e)) raise GameTimeError(errMsg) else: with histogramFileHandler: for binEdge, sample in zip(binEdges, hist): histogramFileHandler.write("%s\t%s\n" % (binEdge, sample)) logger.info("Histogram created.")
def readProjectConfigFile(location): """ Reads project configuration information from the XML file provided. Arguments: location: Location of the XML file that contains project configuration information. Returns: :class:`~gametime.projectConfiguration.ProjectConfiguration` object that contains information from the XML file whose location is provided. """ logger.info("Reading project configuration in %s..." % location) if not os.path.exists(location): errMsg = "Cannot find project configuration file: %s" % location raise GameTimeError(errMsg) try: projectConfigDom = minidom.parse(location) except EnvironmentError as e: errMsg = "Error reading from project configuration file: %s" % e raise GameTimeError(errMsg) # Check that the root element is properly named. rootNode = projectConfigDom.documentElement if rootNode.tagName != 'gametime-project': raise GameTimeError("The root element in the XML file should be " "named `gametime-project'.") # Check that no child element of the root element has an illegal tag. rootChildNodes = [ node for node in rootNode.childNodes if node.nodeType == node.ELEMENT_NODE ] for childNode in rootChildNodes: childNodeTag = childNode.tagName if childNodeTag not in ["file", "preprocess", "analysis", "debug"]: raise GameTimeError("Unrecognized tag: %s" % childNodeTag) # Find the directory that contains the project configuration XML file. projectConfigDir = os.path.dirname(os.path.abspath(location)) # Initialize the instantiation variables for # the ProjectConfiguration object. locationFile, func = "", "" startLabel, endLabel = "", "" included, merged, inlined, unrollLoops = [], [], [], False randomizeInitialBasis = False maximumErrorScaleFactor = 10 determinantThreshold, maxInfeasiblePaths = 0.001, 100 modelAsNestedArrays, preventBasisRefinement = False, False ilpSolverName, smtSolverName = "", "" # Process information about the file to be analyzed. fileNode = (projectConfigDom.getElementsByTagName("file"))[0] for node in fileNode.childNodes: if node.nodeType == node.ELEMENT_NODE: nodeText = _getText(node) nodeTag = node.tagName if nodeTag == "location": locationFile = \ os.path.normpath(os.path.join(projectConfigDir, nodeText)) elif nodeTag == "analysis-function": func = nodeText elif nodeTag == "start-label": startLabel = nodeText elif nodeTag == "end-label": endLabel = nodeText else: raise GameTimeError("Unrecognized tag: %s" % nodeTag) # Process the preprocessing variables and flags. preprocessingNode = \ (projectConfigDom.getElementsByTagName("preprocess"))[0] for node in preprocessingNode.childNodes: if node.nodeType == node.ELEMENT_NODE: nodeText = _getText(node) nodeTag = node.tagName if nodeTag == "unroll-loops": unrollLoops = True elif nodeTag == "include": if nodeText != "": included = getDirPaths(nodeText, projectConfigDir) elif nodeTag == "merge": if nodeText != "": merged = getFilePaths(nodeText, projectConfigDir) elif nodeTag == "inline": if nodeText != "": inlined = getFuncNames(nodeText) else: raise GameTimeError("Unrecognized tag: %s" % nodeTag) # Process the analysis variables and flags. analysisNode = (projectConfigDom.getElementsByTagName("analysis"))[0] for node in analysisNode.childNodes: if node.nodeType == node.ELEMENT_NODE: nodeText = _getText(node) nodeTag = node.tagName if nodeTag == "randomize-initial-basis": randomizeInitialBasis = True elif nodeTag == "maximum-error-scale-factor": maximumErrorScaleFactor = float(nodeText) elif nodeTag == "determinant-threshold": determinantThreshold = float(nodeText) elif nodeTag == "max-infeasible-paths": maxInfeasiblePaths = int(nodeText) elif nodeTag == "model-as-nested-arrays": modelAsNestedArrays = True elif nodeTag == "prevent-basis-refinement": preventBasisRefinement = True elif nodeTag == "ilp-solver": ilpSolverName = nodeText elif nodeTag == "smt-solver": smtSolverName = nodeText else: raise GameTimeError("Unrecognized tag: %s" % nodeTag) # Initialize the instantiation variables for the # DebugConfiguration object. keepCilTemps, dumpIr, keepIlpSolverOutput = False, False, False dumpInstructionTrace, dumpPath, dumpAllPaths = False, False, False dumpSmtTrace, dumpAllQueries = False, False keepParserOutput, keepSimulatorOutput = False, False # Process the debug flags. debugNode = (projectConfigDom.getElementsByTagName("debug"))[0] for node in debugNode.childNodes: if node.nodeType == node.ELEMENT_NODE: nodeText = _getText(node) nodeTag = node.tagName if nodeTag == "keep-cil-temps": keepCilTemps = True elif nodeTag == "dump-ir": dumpIr = True elif nodeTag == "keep-ilp-solver-output": keepIlpSolverOutput = True elif nodeTag == "dump-instruction-trace": dumpInstructionTrace = True elif nodeTag == "dump-path": dumpPath = True elif nodeTag == "dump-all-paths": dumpAllPaths = True elif nodeTag == "dump-smt-trace": dumpSmtTrace = True elif nodeTag == "dump-all-queries": dumpAllQueries = True elif nodeTag == "keep-parser-output": keepParserOutput = True elif nodeTag == "keep-simulator-output": keepSimulatorOutput = True else: raise GameTimeError("Unrecognized tag: %s" % nodeTag) # Instantiate a DebugConfiguration object. debugConfig = DebugConfiguration(keepCilTemps, dumpIr, keepIlpSolverOutput, dumpInstructionTrace, dumpPath, dumpAllPaths, dumpSmtTrace, dumpAllQueries, keepParserOutput, keepSimulatorOutput) # We have obtained all the information we need from the XML # file provided. Instantiate a ProjectConfiguration object. projectConfig = ProjectConfiguration( locationFile, func, smtSolverName, startLabel, endLabel, included, merged, inlined, unrollLoops, randomizeInitialBasis, maximumErrorScaleFactor, determinantThreshold, maxInfeasiblePaths, modelAsNestedArrays, preventBasisRefinement, ilpSolverName, debugConfig) logger.info("Successfully loaded project.") logger.info("") return projectConfig
def findExtremePath(analyzer, extremum=Extremum.LONGEST, interval=None): """Determines either the longest or the shortest path through the DAG with the constraints stored in the ``Analyzer`` object provided. Arguments: analyzer: ``Analyzer`` object that maintains information about the code being analyzed. extremum: Type of extreme path to calculate. interval: ``Interval`` object that represents the interval of values that the generated paths can have. If no ``Interval`` object is provided, the interval of values is considered to be all real numbers. Returns: Tuple whose first element is the longest or the shortest path through the DAG, as a list of nodes along the path (ordered by traversal from source to sink), and whose second element is the integer linear programming problem that was solved to obtain the path, as an object of the ``IlpProblem`` class. If no such path is feasible, given the constraints stored in the ``Analyzer`` object and the ``Interval`` object provided, the first element of the tuple is an empty list, and the second element of the tuple is an ``IlpProblem`` object whose ``objVal`` instance variable is None. """ # Make temporary variables for the frequently accessed # variables from the ``Analyzer`` object provided. projectConfig = analyzer.projectConfig dag = analyzer.dag source = dag.source sink = dag.sink numEdges = dag.numEdges nodesExceptSourceSink = dag.nodesExceptSourceSink edges = dag.allEdges edgeWeights = dag.edgeWeights pathExclusiveConstraints = analyzer.pathExclusiveConstraints pathBundledConstraints = analyzer.pathBundledConstraints # Set up the linear programming problem. logger.info("Setting up the integer linear programming problem...") problem = IlpProblem(_LP_NAME) logger.info("Creating the variables and adding the constraints...") # Set up the variables that correspond to the flow through each edge. # Set each of the variables to be an integer binary variable. edgeFlows = pulp.LpVariable.dicts("EdgeFlow", range(0, numEdges), 0, 1, pulp.LpBinary) # Add a constraint for the flow from the source. The flow through all of # the edges out of the source should sum up to exactly 1. edgeFlowsFromSource = _getEdgeFlowVars(analyzer, edgeFlows, dag.out_edges(source)) problem += pulp.lpSum(edgeFlowsFromSource) == 1, "Flows from source" # Add constraints for the rest of the nodes (except sink). The flow # through all of the edges into a node should equal the flow through # all of the edges out of the node. Hence, for node n, if e_i and e_j # enter a node, and e_k and e_l exit a node, the corresponding flow # equation is e_i + e_j = e_k + e_l. for node in nodesExceptSourceSink: edgeFlowsToNode = _getEdgeFlowVars(analyzer, edgeFlows, dag.in_edges(node)) edgeFlowsFromNode = _getEdgeFlowVars(analyzer, edgeFlows, dag.out_edges(node)) problem += \ (pulp.lpSum(edgeFlowsToNode) == pulp.lpSum(edgeFlowsFromNode), "Flows through %s" % node) # Add a constraint for the flow to the sink. The flow through all of # the edges into the sink should sum up to exactly 1. edgeFlowsToSink = _getEdgeFlowVars(analyzer, edgeFlows, dag.in_edges(sink)) problem += pulp.lpSum(edgeFlowsToSink) == 1, "Flows to sink" # Add constraints for the exclusive path constraints. To ensure that # the edges in each constraint are not taken together, the total flow # through all the edges should add to at least one unit less than # the number of edges in the constraint. Hence, if a constraint # contains edges e_a, e_b, e_c, then e_a + e_b + e_c must be less than 3. # This way, all three of these edges can never be taken together. for constraintNum, path in enumerate(pathExclusiveConstraints): edgeFlowsInConstraint = _getEdgeFlowVars(analyzer, edgeFlows, path) problem += (pulp.lpSum(edgeFlowsInConstraint) <= (len(path) - 1), "Path exclusive constraint %d" % (constraintNum + 1)) # Add constraints for the bundled path constraints. If a constraint # contains edges e_a, e_b, e_c, e_d, and each edge *must* be taken, # then e_b + e_c + e_d must sum up to e_a, scaled by -3 (or one less # than the number of edges in the path constraint). Hence, the flow # constraint is e_b + e_c + e_d = -3 * e_a. By default, we scale # the first edge in a constraint with this negative value. for constraintNum, path in enumerate(pathBundledConstraints): firstEdge = path[0] firstEdgeFlow = _getEdgeFlowVar(analyzer, edgeFlows, firstEdge) edgeFlowsForRest = _getEdgeFlowVars(analyzer, edgeFlows, path[1:]) problem += \ (pulp.lpSum(edgeFlowsForRest) == (len(path)-1) * firstEdgeFlow, "Path bundled constraint %d" % (constraintNum+1)) # There may be bounds on the values of the paths that are generated # by this function: we add constraints for these bounds. For this, # we weight the PuLP variables for the edges using the list of # edge weights provided, and then impose bounds on the sum. weightedEdgeFlowVars = [] for edgeIndex, edgeFlowVar in edgeFlows.items(): edgeWeight = edgeWeights[edgeIndex] weightedEdgeFlowVars.append(edgeWeight * edgeFlowVar) interval = interval or Interval() if interval.hasFiniteLowerBound(): problem += \ (pulp.lpSum(weightedEdgeFlowVars) >= interval.lowerBound) if interval.hasFiniteUpperBound(): problem += \ (pulp.lpSum(weightedEdgeFlowVars) <= interval.upperBound) logger.info("Variables created and constraints added.") logger.info("Constructing the objective function...") # Finally, construct and add the objective function. # We reuse the constraint (possibly) added in the last step of # the constraint addition phase. objective = pulp.lpSum(weightedEdgeFlowVars) problem += objective logger.info("Objective function constructed.") logger.info("Finding the maximum value of the objective function...") problem.sense = pulp.LpMaximize problemStatus = problem.solve(solver=projectConfig.ilpSolver) if problemStatus != pulp.LpStatusOptimal: logger.info("Maximum value not found.") return [], problem objValMax = pulp.value(objective) logger.info("Maximum value found: %g" % objValMax) logger.info("Finding the path that corresponds to the maximum value...") # Determine the edges along the extreme path using the solution. maxPath = [ edges[edgeNum] for edgeNum in edgeFlows if edgeFlows[edgeNum].value() == 1 ] logger.info("Path found.") logger.info("Finding the minimum value of the objective function...") problem.sense = pulp.LpMinimize problemStatus = problem.solve(solver=projectConfig.ilpSolver) if problemStatus != pulp.LpStatusOptimal: logger.info("Minimum value not found.") return [], problem objValMin = pulp.value(objective) logger.info("Minimum value found: %g" % objValMin) logger.info("Finding the path that corresponds to the minimum value...") # Determine the edges along the extreme path using the solution. minPath = [ edges[edgeNum] for edgeNum in edgeFlows if edgeFlows[edgeNum].value() == 1 ] logger.info("Path found.") # Choose the correct extreme path based on the optimal solutions # and the type of extreme path required. absMax, absMin = abs(objValMax), abs(objValMin) if extremum is Extremum.LONGEST: extremePath = maxPath if absMax >= absMin else minPath problem.sense = (pulp.LpMaximize if absMax >= absMin else pulp.LpMinimize) problem.objVal = max(absMax, absMin) elif extremum is Extremum.SHORTEST: extremePath = minPath if absMax >= absMin else maxPath problem.sense = (pulp.LpMinimize if absMax >= absMin else pulp.LpMaximize) problem.objVal = min(absMax, absMin) # Arrange the nodes along the extreme path in order of traversal # from source to sink. resultPath = [] logger.info("Arranging the nodes along the chosen extreme path " "in order of traversal...") # To do so, first construct a dictionary mapping a node along the path # to the edge from that node. extremePathDict = {} for edge in extremePath: extremePathDict[edge[0]] = edge # Now, "thread" a path through the dictionary. currNode = source resultPath.append(currNode) while currNode in extremePathDict: newEdge = extremePathDict[currNode] currNode = newEdge[1] resultPath.append(currNode) logger.info("Nodes along the chosen extreme path arranged.") if projectConfig.debugConfig.KEEP_ILP_SOLVER_OUTPUT: _moveIlpFiles(os.getcwd(), projectConfig.locationTempDir) else: _removeTempIlpFiles() # We're done! return resultPath, problem
def findMinimalOvercompleteBasis(analyzer, paths, k): """ This function is here only for test purposes. The functions finds the smallest set of 'basis paths' with the following property: Each path pi in `paths', can be expressed as a linear combination pi = a_1 b_1 + ... + a_n b_n of paths b_i from `basis`. This function finds the set of basis paths such that every path can be expressed as a linear combination of basis paths b_i such that the sum of absolute value of coefficients is at most 'k': |a_1| + |a_2| + ... + |a_n| <= k Arguments analyzer: ``Analyzer`` object that maintains information about the code being analyzed. paths: List of paths that we want to find out how well can be expressed as a linear combination of paths in `basis` k: bound on how well the 'paths' can be expressed as a linear combination of the calculated basis paths Returns: List of paths satisfying the condition stated above """ dag = analyzer.dag source = dag.soure sink = dag.sink numEdges = dag.numEdges edges = dag.edges() numPaths = len(paths) # Set up the linear programming problem. logger.info("Number of paths: %d " % numPaths) logger.info("Setting up the integer linear programming problem...") problem = IlpProblem(_LP_NAME) logger.info("Creating variables") indices = [(i, j) for i in range(numPaths) for j in range(numPaths)] coeffs = pulp.LpVariable.dicts("c", indices, -k, k) absValues = pulp.LpVariable.dicts("abs", indices, 0, k) usedPaths = pulp.LpVariable.dicts("used", range(numPaths), 0, 1, pulp.LpBinary) logger.info("Adding usedPaths") for i in range(numPaths): for j in range(numPaths): problem += k * usedPaths[j] >= absValues[(i, j)] logger.info("Add absolute values") for index in indices: problem += absValues[index] >= coeffs[index] problem += absValues[index] >= -coeffs[index] for i in range(numPaths): logger.info("Processing path number %d" % i) # all coefficients expressing path i allCoeffExpressing = [absValues[(i, j)] for j in range(numPaths)] problem += pulp.lpSum(allCoeffExpressing) <= k for edge in edges: pathsContainingEdge = \ [j for j in range(numPaths) if (edge in paths[j])] presentCoeffs = [coeffs[(i, j)] for j in pathsContainingEdge] present = 1 if (edge in paths[i]) else 0 problem += pulp.lpSum(presentCoeffs) == present objective = pulp.lpSum(usedPaths) problem += objective problem.sense = pulp.LpMinimize problemStatus = problem.solve(solver=projectConfig.ilpSolver) if problemStatus != pulp.LpStatusOptimal: logger.info("Minimum value not found.") return [], problem objValMin = pulp.value(objective) logger.info("Minimum value found: %g" % objValMin) solutionPaths = \ [index for index in range(numPaths) if usedPaths[index].value() == 1] return solutionPaths
def findGoodnessOfFit(analyzer, paths, basis): """ This function is here only for test purposes. Each path pi in `paths', can be expressed as a linear combination pi = a_1 b_1 + ... + a_n b_n of paths b_i from `basis`. This function returns the least number `c` such that every path can be expressed as a linear combination of basis paths b_i such that the sum of absolute value of coefficients is at most `c`: |a_1| + |a_2| + ... + |a_n| <= c Arguments analyzer: ``Analyzer`` object that maintains information about the code being analyzed. paths: List of paths that we want to find out how well can be expressed as a linear combination of paths in `basis` basis: List of paths that are used to express `paths` as a linear combination of Returns: The number `c` as described in the paragraph above. """ dag = analyzer.dag source = dag.source sink = dag.sink numEdges = dag.numEdges edges = dag.edges() numPaths = len(paths) numBasis = len(basis) projectConfig = analyzer.projectConfig # Set up the linear programming problem. logger.info("Number of paths: %d " % numPaths) logger.info("Number of basis paths: %d " % numBasis) logger.info("Setting up the integer linear programming problem...") problem = IlpProblem("BLAH") logger.info("Creating variables") indices = [(i, j) for i in range(numPaths) for j in range(numBasis)] coeffs = pulp.LpVariable.dicts("c", indices, -100, 100) absValues = pulp.LpVariable.dicts("abs", indices, 0, 100) bound = pulp.LpVariable("bnd", 0, 10000) logger.info("Add absolute values") for index in indices: problem += absValues[index] >= coeffs[index] problem += absValues[index] >= -coeffs[index] for i in range(numPaths): # all coefficients expressing path i allCoeffExpressing = [absValues[(i, j)] for j in range(numBasis)] problem += pulp.lpSum(allCoeffExpressing) <= bound # express path i as a linear combination of basis paths for edge in edges: pathsContainingEdge = \ [j for j in range(numBasis) if (edge in basis[j])] presentCoeffs = [coeffs[(i, j)] for j in pathsContainingEdge] present = 1 if (edge in paths[i]) else 0 problem += pulp.lpSum(presentCoeffs) == present problem += bound problem.sense = pulp.LpMinimize problemStatus = problem.solve(solver=projectConfig.ilpSolver) if problemStatus != pulp.LpStatusOptimal: logger.info("Minimum value not found.") return [], problem objValMin = pulp.value(bound) logger.info("Minimum value found: %g" % objValMin) return objValMin
def generateAndSolveCoreProblem(analyzer, paths, pathFunctionUpper, pathFunctionLower, weightsPositive, printProblem=False, extremum=Extremum.LONGEST): """This function actually constructs the ILP to find the longest path in the graph specified by 'analyzer' using the set of measured paths given by 'paths'. Arguments analyzer: ``Analyzer`` object that maintains information about the code being analyzed. Among others, contains the underlying DAG or the collection of infeasible paths. paths: List of paths used in the measurements. Each path is a list of edges in the order in which they are visited by the path pathFunctionUpper: Function of type: path -> float that for a given path should return the upper bound on the length of the given path. The input 'path' is always from 'paths' pathFunctionLower: Function of type: path -> float that for a given path should return the upper bound on the length of the given path. The input 'path' is always from 'paths' weightsPositive: Boolean value specifying whether the individual edge weight are required to be at least 0 (if set to True) or can be arbitrary real value (if set to False) printProblem: Boolean value used for debugging. If set to true, the generated ILP is printed. extremum: Specifies whether we are calculating Extremum.LONGEST or Extremum.SHORTEST Returns: Triple consisting of the length of the longest path found, the actual path and the ILP problem generated. """ dag = analyzer.dag dag.initializeDictionaries() source = dag.source sink = dag.sink numEdges = dag.numEdges edges = dag.edges() numPaths = len(paths) # Use the compact representation of the DAG # compact is now a mapping that for each edge of dag gives an index of an # edge in the compact graph. compact = makeCompact(dag) projectConfig = analyzer.projectConfig nodesExceptSourceSink = dag.nodesExceptSourceSink pathExclusiveConstraints = analyzer.pathExclusiveConstraints pathBundledConstraints = analyzer.pathBundledConstraints # Set up the linear programming problem. logger.info("Number of paths: %d " % numPaths) logger.info("Setting up the integer linear programming problem...") problem = IlpProblem(_LP_NAME) # Take M to be the maximum edge length. Add 1.0 to make sure there are # no problems due to rounding errors. M = max([pathFunctionUpper(path) for path in paths] + [0]) + 1.0 if not weightsPositive: M *= numEdges logger.info("Using value %.2f for M --- the maximum edge weight" % M) logger.info("Creating variables") values = set() for key in compact: values.add(compact[key]) new_edges = len(values) # Set up the variables that correspond to the flow through each edge. # Set each of the variables to be an integer binary variable. edgeFlows = pulp.LpVariable.dicts("EdgeFlow", range(0, new_edges), 0, 1, pulp.LpBinary) edgeWeights = pulp.LpVariable.dicts("we", range(0, new_edges), 0 if weightsPositive else -M, M) # for a given 'path' in the original DAG returns the edgeFlow variables # corresponding to the edges along the same path in the compact DAG. def getNewIndices(compact, edgeFlows, path): edges = [compact[edge] for edge in path] pathWeights = [edgeFlows[edge] for edge in set(edges)] return pathWeights for path in paths: pathWeights = \ getNewIndices(compact, edgeWeights, dag.getEdges(path.nodes)) problem += pulp.lpSum(pathWeights) <= pathFunctionUpper(path) problem += pulp.lpSum(pathWeights) >= pathFunctionLower(path) # Add a constraint for the flow from the source. The flow through all of # the edges out of the source should sum up to exactly 1. edgeFlowsFromSource = \ getNewIndices(compact, edgeFlows, dag.out_edges(source)) problem += pulp.lpSum(edgeFlowsFromSource) == 1, "Flows from source" # Add constraints for the rest of the nodes (except sink). The flow # through all of the edges into a node should equal the flow through # all of the edges out of the node. Hence, for node n, if e_i and e_j # enter a node, and e_k and e_l exit a node, the corresponding flow # equation is e_i + e_j = e_k + e_l. for node in nodesExceptSourceSink: if (dag.neighbors(node) == 1) and (dag.predecessors(node) == 1): continue edgeFlowsToNode = getNewIndices(compact, edgeFlows, dag.in_edges(node)) edgeFlowsFromNode = getNewIndices(compact, edgeFlows, dag.out_edges(node)) problem += \ (pulp.lpSum(edgeFlowsToNode) == pulp.lpSum(edgeFlowsFromNode), "Flows through %s" % node) # Add a constraint for the flow to the sink. The flow through all of # the edges into the sink should sum up to exactly 1. edgeFlowsToSink = getNewIndices(compact, edgeFlows, dag.in_edges(sink)) problem += pulp.lpSum(edgeFlowsToSink) == 1, "Flows to sink" # Add constraints for the exclusive path constraints. To ensure that # the edges in each constraint are not taken together, the total flow # through all the edges should add to at least one unit less than # the number of edges in the constraint. Hence, if a constraint # contains edges e_a, e_b, e_c, then e_a + e_b + e_c must be less than 3. # This way, all three of these edges can never be taken together. for constraintNum, path in enumerate(pathExclusiveConstraints): edgeFlowsInConstraint = getNewIndices(compact, edgeFlows, path) problem += (pulp.lpSum(edgeFlowsInConstraint) <= (len(edgeFlowsInConstraint) - 1), "Path exclusive constraint %d" % (constraintNum + 1)) # Each productVars[index] in the longest path should correspond to # edgeFlows[index] * edgeWeights[index] productVars = pulp.LpVariable.dicts("pe", range(0, new_edges), -M, M) for index in range(0, new_edges): if extremum == Extremum.LONGEST: problem += productVars[index] <= edgeWeights[index] problem += productVars[index] <= M * edgeFlows[index] else: problem += productVars[index] >= edgeWeights[index] - M * ( 1.0 - edgeFlows[index]) problem += productVars[index] >= 0 objective = pulp.lpSum(productVars) problem += objective logger.info("Objective function constructed.") if extremum == Extremum.LONGEST: logger.info("Finding the maximum value of the objective function...") problem.sense = pulp.LpMaximize else: logger.info("Finding the minimum value of the objective function...") problem.sense = pulp.LpMinimize problemStatus = problem.solve(solver=projectConfig.ilpSolver) if (printProblem): logger.info(problem) if problemStatus != pulp.LpStatusOptimal: logger.info("Maximum value not found.") return -1, [], problem objValMax = pulp.value(objective) problem.objVal = objValMax logger.info("Maximum value found: %g" % objValMax) logger.info("Finding the path that corresponds to the maximum value...") # Determine the edges along the extreme path using the solution. maxPath = [ edges[edgeNum] for edgeNum in edgeFlows if edgeFlows[edgeNum].value() > 0.1 ] logger.info("Path found.") totalLength = sum([ productVars[edgeNum].value() for edgeNum in edgeFlows if edgeFlows[edgeNum].value() == 1 ]) logger.info("Total length of the path %.2f" % totalLength) objValMax = totalLength maxPath = [ edgeNum for edgeNum in range(0, new_edges) if edgeFlows[edgeNum].value() > 0.1 ] extremePath = [] #reverse exremePath according to the compact edgeMap for edge in maxPath: map_to = [source for source in compact if compact[source] == edge] extremePath.extend(map_to) # Arrange the nodes along the extreme path in order of traversal # from source to sink. resultPath = [] logger.info("Arranging the nodes along the chosen extreme path " "in order of traversal...") # To do so, first construct a dictionary mapping a node along the path # to the edge from that node. extremePathDict = {} for edge in extremePath: extremePathDict[edge[0]] = edge # Now, "thread" a path through the dictionary. currNode = dag.source resultPath.append(currNode) while currNode in extremePathDict: newEdge = extremePathDict[currNode] currNode = newEdge[1] resultPath.append(currNode) logger.info("Nodes along the chosen extreme path arranged.") if projectConfig.debugConfig.KEEP_ILP_SOLVER_OUTPUT: _moveIlpFiles(os.getcwd(), projectConfig.locationTempDir) else: _removeTempIlpFiles() # We're done! return objValMax, resultPath, problem
def findLeastCompatibleMuMax(analyzer, paths): """This function returns the least dealta in the underlying graph, as specified by 'analyzer', that is feasible with the given set of measurements as specified by 'paths'. The method does not take into account which paths are feasible and which not; it considers all the paths in the graph. Arguments: analyzer: ``Analyzer`` object that maintains information about the code being analyzed. paths: List of paths used in the measurements. Each path is a list of edges in the order in which they are visited by the path Returns: A floting point value---the least delta compatible with the measurements """ dag = analyzer.dag source = dag.source sink = dag.sink numEdges = dag.numEdges edges = dag.edges() numPaths = len(paths) projectConfig = analyzer.projectConfig nodesExceptSourceSink = dag.nodesExceptSourceSink # Set up the linear programming problem. logger.info("Number of paths: %d " % numPaths) logger.info("Setting up the integer linear programming problem...") problem = IlpProblem(_LP_NAME) logger.info("Creating variables") # Set up the variables that correspond to weights of each edge. # Each edge is restricted to be a nonnegative real number edgeWeights = pulp.LpVariable.dicts("we", range(0, numEdges), 0) # Create the variable that shall correspond to the least delta delta = pulp.LpVariable("delta", 0) for path in paths: pathWeights = \ _getEdgeFlowVars(analyzer, edgeWeights, dag.getEdges(path.nodes)) problem += pulp.lpSum(pathWeights) <= delta + path.measuredValue problem += pulp.lpSum(pathWeights) >= -delta + path.measuredValue print "LENGTH:", path.measuredValue # Optimize for the least delta problem += delta logger.info("Finding the minimum value of the objective function...") problem.sense = pulp.LpMinimize problemStatus = problem.solve(solver=projectConfig.ilpSolver) if problemStatus != pulp.LpStatusOptimal: logger.info("Maximum value not found.") return [] objValMin = pulp.value(delta) logger.info("Minimum compatible delta found: %g" % objValMin) if projectConfig.debugConfig.KEEP_ILP_SOLVER_OUTPUT: _moveIlpFiles(os.getcwd(), projectConfig.locationTempDir) else: _removeTempIlpFiles() return objValMin
def generatePaths(analyzer, numPaths=5, pathType=PathType.WORST_CASE, interval=None, useObExtraction=False): """Generates a list of feasible paths of the code being analyzed, each represented by an object of the ``Path`` class. The type of the generated paths is determined by the pathType argument, which is a class variable of the ``PathType`` class. By default, this argument is ``PathType.WORST_CASE``. For a description of the types, refer to the documentation of the ``PathType`` class. The ``numPaths`` argument is an upper bound on how many paths should be generated, which is 5 by default. This argument is ignored if this method is used to generate all of the feasible paths of the code being analyzed. The ``interval`` argument is an ``Interval`` object that represents the interval of values that the generated paths can have. If no ``Interval`` object is provided, the interval of values is considered to be all real numbers. This method is idempotent: a second call to this method will produce the same list of ``Path`` objects as the first call, assuming that nothing has changed between the two calls. Precondition: The basis ``Path`` objects of the input ``Analyzer`` object have values associated with them. Refer to either the method ``loadBasisValuesFromFile`` or the method ``loadBasisValues`` in the ``Analyzer`` class. Arguments: analyzer: ``Analyzer`` object that maintains information about the code being analyzed. numPaths: Upper bound on the number of paths to generate. pathType: Type of paths to generate, represented by a class variable of the ``PathType`` class. The different types of paths are described in the documentation of the ``PathType`` class. interval: ``Interval`` object that represents the interval of values that the generated paths can have. If no ``Interval`` object is provided, the interval of values is considered to be all real numbers. useObExtraction: Boolean value specifiying whether to use overcomplete basis extraction algorithm Returns: List of feasible paths of the code being analyzed, each represented by an object of the ``Path`` class. """ paths = None if pathType == PathType.WORST_CASE: logger.info("Generating %d worst-case feasible paths..." % numPaths) paths = \ PathGenerator._generatePaths(analyzer, numPaths, pulpHelper.Extremum.LONGEST, interval, useObExtraction) elif pathType == PathType.BEST_CASE: logger.info("Generating %d best-case feasible paths..." % numPaths) paths = \ PathGenerator._generatePaths(analyzer, numPaths, pulpHelper.Extremum.SHORTEST, interval, useObExtraction) if paths is not None: logger.info("%d of %d paths have been generated." % (len(paths), numPaths)) return paths if pathType == PathType.ALL_DECREASING: logger.info("Generating all feasible paths in decreasing order " "of value...") paths = \ PathGenerator._generatePaths(analyzer, analyzer.dag.numPaths, pulpHelper.Extremum.LONGEST, interval, useObExtraction) elif pathType == PathType.ALL_INCREASING: logger.info("Generating all feasible paths in increasing order " "of value...") paths = \ PathGenerator._generatePaths(analyzer, analyzer.dag.numPaths, pulpHelper.Extremum.SHORTEST, interval, useObExtraction) if paths is not None: logger.info("%d feasible paths have been generated." % len(paths)) return paths if pathType == PathType.RANDOM: logger.info("Generating random feasible paths...") paths = \ PathGenerator._generatePaths(analyzer, numPaths, None, interval) logger.info("%d of %d paths have been generated." % (len(paths), numPaths)) return paths else: raise GameTimeError("Unrecognized path type: %d" % pathType)
def _generatePaths(analyzer, numPaths, extremum=pulpHelper.Extremum.LONGEST, interval=None, useObExtraction=False): """Helper static method for the ``generatePaths`` static method. Generates a list of feasible paths of the code being analyzed, each represented by an object of the ``Path`` class. Arguments: analyzer: ``Analyzer`` object that maintains information about the code being analyzed. numPaths: Upper bound on the number of paths to generate. extremum: Type of paths to calculate (longest or shortest), represented by an element of the ``Extremum`` class in the ``pulpHelper`` module. interval: ``Interval`` object that represents the interval of values that the generated paths can have. If no ``Interval`` object is provided, the interval of values is considered to be all real numbers. useObExtraction: Boolean value specifiying whether to use overcomplete basis extraction algorithm Returns: List of feasible paths of the code being analyzed, each represented by an object of the ``Path`` class. """ if nxHelper.hasCycles(analyzer.dag): logger.warn("Loops in the code have been detected.") logger.warn("No feasible paths have been generated.") return [] logger.info("") startTime = time.clock() if useObExtraction: beforeTime = time.clock() logger.info("Using the new algorithm to extract the longest path") logger.info("Finding Least Compatible Delta") muMax = pulpHelper.findLeastCompatibleMuMax( analyzer, analyzer.basisPaths) logger.info("Found the least mu_max compatible with measurements: " "%.2f in %.2f seconds" % (muMax, time.clock() - beforeTime)) analyzer.inferredMuMax = muMax beforeTime = time.clock() logger.info("Calculating error bounds in the estimate") analyzer.errorScaleFactor, path, ilpProblem = \ pulpHelper.findWorstExpressiblePath( analyzer, analyzer.basisPaths, 0) logger.info( "Total maximal error in estimates is 2 x %.2f x %.2f = %.2f" % (analyzer.errorScaleFactor, muMax, 2 * analyzer.errorScaleFactor * muMax)) logger.info("Calculated in %.2f ms" % (time.clock() - beforeTime)) else: analyzer.estimateEdgeWeights() resultPaths = [] currentPathNum, numPathsUnsat, numCandidatePaths = 0, 0, 0 while (currentPathNum < numPaths and numCandidatePaths < analyzer.dag.numPaths): logger.info("Currently generating path %d..." % (currentPathNum + 1)) logger.info("So far, %d candidate paths were found to be " "unsatisfiable." % numPathsUnsat) if analyzer.pathDimension == 1: warnMsg = ("Basis matrix has dimensions 1x1. " "There is only one path through the function " "under analysis.") logger.warn(warnMsg) logger.info("Finding a candidate path using an integer " "linear program...") logger.info("") if extremum is None: source, sink = analyzer.dag.source, analyzer.dag.sink candidatePathNodes = nxHelper.getRandomPath( analyzer.dag, source, sink) candidatePathEdges = Dag.getEdges(candidatePathNodes) analyzer.addPathBundledConstraint(candidatePathEdges) if useObExtraction: candidatePathNodes, ilpProblem = \ pulpHelper.findLongestPathWithDelta( analyzer, analyzer.basisPaths, muMax, extremum) else: candidatePathNodes, ilpProblem = \ pulpHelper.findExtremePath(analyzer, extremum if extremum is not None else pulpHelper.Extremum.LONGEST, interval) logger.info("") if ilpProblem.objVal is None: if (extremum is not None or numCandidatePaths == analyzer.dag.numPaths): logger.info("Unable to find a new candidate path.") break elif extremum is None: analyzer.addPathExclusiveConstraint(candidatePathEdges) analyzer.resetPathBundledConstraints() numCandidatePaths = len(analyzer.pathExclusiveConstraints) continue logger.info("Candidate path found.") candidatePathEdges = Dag.getEdges(candidatePathNodes) candidatePathValue = ilpProblem.objVal logger.info("Checking if candidate path is feasible...") logger.info("") resultPath = analyzer.checkFeasibility(candidatePathNodes, ilpProblem) querySatisfiability = resultPath.smtQuery.satisfiability if querySatisfiability == Satisfiability.SAT: logger.info("Candidate path is feasible.") resultPath.setPredictedValue(candidatePathValue) resultPaths.append(resultPath) logger.info("Path %d generated." % (currentPathNum + 1)) # Exclude the path generated from future iterations. analyzer.addPathExclusiveConstraint(candidatePathEdges) currentPathNum += 1 numPathsUnsat = 0 elif querySatisfiability == Satisfiability.UNSAT: logger.info("Candidate path is infeasible.") logger.info("Finding the edges to exclude...") unsatCore = resultPath.smtQuery.unsatCore excludeEdges = resultPath.getEdgesForConditions(unsatCore) logger.info("Edges to be excluded found.") logger.info("Adding constraint to exclude these edges...") if len(excludeEdges) > 0: analyzer.addPathExclusiveConstraint(excludeEdges) else: analyzer.addPathExclusiveConstraint(candidatePathEdges) logger.info("Constraint added.") numPathsUnsat += 1 numCandidatePaths += 1 if extremum is None: analyzer.resetPathBundledConstraints() logger.info("") logger.info("") analyzer.resetPathExclusiveConstraints() logger.info("Time taken to generate paths: %.2f seconds." % (time.clock() - startTime)) return resultPaths