def evaluateWithMarabou(self, inputValues, filename="evaluateWithMarabou.log", options=None): """ Function to evaluate network at a given point using Marabou as solver Arguments: inputValues: list of (np arrays) representing input to network filename: (string) path to redirect output Returns: outputValues: (np array) representing output of network """ inputVars = self.inputVars # list of numpy arrays outputVars = self.outputVars inputDict = dict() inputVarList = np.concatenate(inputVars, axis=-1).ravel() inputValList = np.concatenate(inputValues).ravel() assignList = zip(inputVarList, inputValList) for x in assignList: inputDict[x[0]] = x[1] ipq = self.getMarabouQuery() for k in inputDict: ipq.setLowerBound(k, inputDict[k]) ipq.setUpperBound(k, inputDict[k]) if options == None: options = MarabouCore.Options() outputDict = MarabouCore.solve(ipq, options, filename) outputValues = outputVars.reshape(-1).astype(np.float64) for i in range(len(outputValues)): outputValues[i] = (outputDict[0])[outputValues[i]] outputValues = outputValues.reshape(outputVars.shape) return outputValues
def solve(self, filename="", verbose=True, options=None): """Function to solve query represented by this network Args: filename (string): Path for redirecting output verbose (bool): If true, print out solution after solve finishes options (:class:`~maraboupy.MarabouCore.Options`): Object for specifying Marabou options, defaults to None Returns: (tuple): tuple containing: - exitCode (str): A string representing the exit code (sat/unsat/TIMEOUT/ERROR/UNKNOWN/QUIT_REQUESTED). - vals (Dict[int, float]): Empty dictionary if UNSAT, otherwise a dictionary of SATisfying values for variables - stats (:class:`~maraboupy.MarabouCore.Statistics`): A Statistics object to how Marabou performed """ ipq = self.getMarabouQuery() if options == None: options = MarabouCore.Options() exitCode, vals, stats = MarabouCore.solve(ipq, options, str(filename)) if verbose: print(exitCode) if exitCode == "sat": for j in range(len(self.inputVars)): for i in range(self.inputVars[j].size): print("input {} = {}".format( i, vals[self.inputVars[j].item(i)])) for j in range(len(self.outputVars)): for i in range(self.outputVars[j].size): print("output {} = {}".format( i, vals[self.outputVars[j].item(i)])) return [exitCode, vals, stats]
def solve(self, filename="", verbose=True, options=None): """ Function to solve query represented by this network Arguments: filename: (string) path to redirect output to verbose: (bool) whether to print out solution after solve finishes options: (MarabouCore.Options) object for specifying Marabou options Returns: vals: (dict: int->float) empty if UNSAT, else SATisfying solution stats: (Statistics) a Statistics object as defined in Marabou, it has multiple methods that provide information related to how an input query was solved. """ ipq = self.getMarabouQuery() if options == None: options = MarabouCore.Options() vals, stats = MarabouCore.solve(ipq, options, filename) if verbose: if stats.hasTimedOut(): print("TO") elif len(vals) == 0: print("unsat") else: print("sat") for j in range(len(self.inputVars)): for i in range(self.inputVars[j].size): print("input {} = {}".format( i, vals[self.inputVars[j].item(i)])) for i in range(self.outputVars.size): print("output {} = {}".format( i, vals[self.outputVars.item(i)])) return [vals, stats]
def solve(self, filename="", verbose=True, options=None): """Function to solve query represented by this network Args: filename (string): Path for redirecting output verbose (bool): If true, print out solution after solve finishes options (:class:`~maraboupy.MarabouCore.Options`): Object for specifying Marabou options, defaults to None Returns: (tuple): tuple containing: - vals (Dict[int, float]): Empty dictionary if UNSAT, otherwise a dictionary of SATisfying values for variables - stats (:class:`~maraboupy.MarabouCore.Statistics`): A Statistics object to how Marabou performed """ ipq = self.getMarabouQuery() if options == None: options = MarabouCore.Options() vals, stats = MarabouCore.solve(ipq, options, filename) if verbose: if stats.hasTimedOut(): print("TO") elif len(vals) == 0: print("unsat") else: print("sat") for j in range(len(self.inputVars)): for i in range(self.inputVars[j].size): print("input {} = {}".format( i, vals[self.inputVars[j].item(i)])) for i in range(self.outputVars.size): print("output {} = {}".format( i, vals[self.outputVars.item(i)])) return [vals, stats]
def evaluateWithMarabou(self, inputValues, filename="evaluateWithMarabou.log", options=None): """Function to evaluate network at a given point using Marabou as solver Args: inputValues (list of np arrays): Inputs to evaluate filename (str): Path to redirect output if using Marabou solver, defaults to "evaluateWithMarabou.log" options (:class:`~maraboupy.MarabouCore.Options`): Object for specifying Marabou options, defaults to None Returns: (list of np arrays): Values representing the outputs of the network or None if system is UNSAT """ # Make sure inputValues is a list of np arrays and not list of lists inputValues = [np.array(inVal) for inVal in inputValues] inputVars = self.inputVars # list of numpy arrays outputVars = self.outputVars # list of numpy arrays inputDict = dict() inputVarList = np.concatenate([inVar.flatten() for inVar in inputVars], axis=-1).flatten() inputValList = np.concatenate( [inVal.flatten() for inVal in inputValues]).flatten() assignList = zip(inputVarList, inputValList) for x in assignList: inputDict[x[0]] = x[1] ipq = self.getMarabouQuery() for k in inputDict: ipq.setLowerBound(k, inputDict[k]) ipq.setUpperBound(k, inputDict[k]) if options == None: options = MarabouCore.Options() exitCode, outputDict, _ = MarabouCore.solve(ipq, options, str(filename)) # When the query is UNSAT an empty dictionary is returned if outputDict == {}: return None outputValues = [ outVars.reshape(-1).astype(np.float64) for outVars in outputVars ] for i in range(len(outputValues)): for j in range(len(outputValues[i])): outputValues[i][j] = outputDict[outputValues[i][j]] outputValues[i] = outputValues[i].reshape(outputVars[i].shape) return outputValues
def solve(self, filename="", verbose=True, options=None): """ Function to solve query represented by this network Arguments: filename: (string) path to redirect output to verbose: (bool) whether to print out solution after solve finishes timeout: (int) time in seconds when Marabou will time out verbosity: (int) determines how much Marabou prints during solving 0: print out minimal information 1: print out statistics only in the beginning and the end 2: print out statistics during solving Returns: vals: (dict: int->float) empty if UNSAT, else SATisfying solution stats: (Statistics) a Statistics object as defined in Marabou, it has multiple methods that provide information related to how an input query was solved. """ ipq = self.getMarabouQuery() if options == None: options = MarabouCore.Options() vals, stats = MarabouCore.solve(ipq, options, filename) if verbose: if stats.hasTimedOut(): print("TO") elif len(vals) == 0: print("UNSAT") else: print("SAT") for inputVarArray in self.inputVars: for inputVar in inputVarArray.flatten(): print("input {} = {}".format(inputVar, vals[inputVar])) # print("input var {} input {} = {}".format(i, self.inputVars[j][0][i],vals[self.inputVars[j].item(i)])) # for j in range(len(self.inputVars)): # for i in range(self.inputVars[j].size): # print("input {} = {}".format(i, vals[self.inputVars[j].item(i)])) # print("input var {} input {} = {}".format(i, self.inputVars[j][0][i],vals[self.inputVars[j].item(i)])) for i in range(self.outputVars.size): print("output {} = {}".format( i, vals[self.outputVars.item(i)])) return [vals, stats]
def evaluateWithMarabou(self, inputValues, filename="evaluateWithMarabou.log", options=None): """ Function to evaluate network at a given point using Marabou as solver Arguments: inputValues: list of (np arrays) representing input to network filename: (string) path to redirect output options: (MarabouCore.Options) object for specifying Marabou options Returns: outputValues: (np array) representing output of network """ # Make sure inputValues is a list of np arrays and not list of lists inputValues = [np.array(inVal) for inVal in inputValues] inputVars = self.inputVars # list of numpy arrays outputVars = self.outputVars inputDict = dict() inputVarList = np.concatenate([inVar.flatten() for inVar in inputVars], axis=-1).flatten() inputValList = np.concatenate( [inVal.flatten() for inVal in inputValues]).flatten() assignList = zip(inputVarList, inputValList) for x in assignList: inputDict[x[0]] = x[1] ipq = self.getMarabouQuery() for k in inputDict: ipq.setLowerBound(k, inputDict[k]) ipq.setUpperBound(k, inputDict[k]) if options == None: options = MarabouCore.Options() outputDict, _ = MarabouCore.solve(ipq, options, filename) outputValues = outputVars.reshape(-1).astype(np.float64) for i in range(len(outputValues)): outputValues[i] = outputDict[outputValues[i]] outputValues = outputValues.reshape(outputVars.shape) return outputValues
def evaluateLocalRobustness(self, input, epsilon, originalClass, verbose=True, options=None, targetClass=None): """Function evaluating a specific input is a local robustness within the scope of epslion Args: input (numpy.ndarray): Target input epsilon (float): L-inf norm of purturbation originalClass (int): Output class of a target input verbose (bool): If true, print out solution after solve finishes options (:class:`~maraboupy.MarabouCore.Options`): Object for specifying Marabou options, defaults to None targetClass (int): If set, find a feasible solution with which the value of targetClass is max within outputs. Returns: (tuple): tuple containing: - vals (Dict[int, float]): Empty dictionary if UNSAT, otherwise a dictionary of SATisfying values for variables - stats (:class:`~maraboupy.MarabouCore.Statistics`): A Statistics object to how Marabou performed - maxClass (int): Output class which value is max within outputs if SAT. """ inputVars = None if (type(self.inputVars) is list): if (len(self.inputVars) != 1): raise NotImplementedError( "Operation for %d inputs is not implemented" % len(self.inputVars)) inputVars = self.inputVars[0][0] elif (type(self.inputVars) is np.ndarray): inputVars = self.inputVars[0] else: err_msg = "Unpexpected type of input vars." raise RuntimeError(err_msg) if inputVars.shape != input.shape: raise RuntimeError( "Input shape of the model should be same as the input shape\n input shape of the model: {0}, shape of the input: {1}" .format(inputVars.shape, input.shape)) if (type(self.outputVars) is list): if (len(self.outputVars) != 1): raise NotImplementedError( "Operation for %d outputs is not implemented" % len(self.outputVars)) elif (type(self.outputVars) is np.ndarray): if (len(self.outputVars) != 1): raise NotImplementedError( "Operation for %d outputs is not implemented" % len(self.outputVars)) else: err_msg = "Unpexpected type of output vars." raise RuntimeError(err_msg) if options == None: options = MarabouCore.Options() # Add constratins to all input nodes flattenInputVars = inputVars.flatten() flattenInput = input.flatten() for i in range(flattenInput.size): self.setLowerBound(flattenInputVars[i], flattenInput[i] - epsilon) self.setUpperBound(flattenInputVars[i], flattenInput[i] + epsilon) maxClass = None outputStartIndex = self.outputVars[0][0][0] if targetClass is None: outputLayerSize = len(self.outputVars[0][0]) # loop for all of output classes except for original class for outputLayerIndex in range(outputLayerSize): if outputLayerIndex != originalClass: self.addMaxConstraint( set([ outputStartIndex + outputLayerIndex, outputStartIndex + originalClass ]), outputStartIndex + outputLayerIndex) exitCode, vals, stats = self.solve(options=options) if (stats.hasTimedOut()): break elif (len(vals) > 0): maxClass = outputLayerIndex break else: self.addMaxConstraint(set(self.outputVars[0][0]), outputStartIndex + targetClass) exitCode, vals, stats = self.solve(options=options) if verbose: if not stats.hasTimedOut() and len(vals) > 0: maxClass = targetClass # print timeout, or feasible inputs and outputs if verbose is on. if verbose: if stats.hasTimedOut(): print("TO") elif len(vals) > 0: print("sat") for j in range(len(self.inputVars[0])): for i in range(self.inputVars[0][j].size): print("input {} = {}".format( i, vals[self.inputVars[0][j].item(i)])) for j in range(len(self.outputVars[0])): for i in range(self.outputVars[0][j].size): print("output {} = {}".format( i, vals[self.outputVars[0][j].item(i)])) return [vals, stats, maxClass]
def verify(self, network: networks.NeuralNetwork, prop: Property) -> (bool, typing.Optional[Tensor.Tensor]): """ Verify that the neural network of interest satisfy the property given as argument using the Marabou verification tool. Parameters ---------- network : NeuralNetwork The neural network to train. prop : Dataset The property which the neural network must satisfy. Returns ---------- (bool, Optional[Tensor]) True and None if the neural network satisfy the property, False and the counterexample otherwise. """ if isinstance(prop, SMTLIBProperty): targeted, bounds, target = utilities.parse_linf_robustness_smtlib(prop.smtlib_path) elif isinstance(prop, LocalRobustnessProperty): targeted = prop.targeted target = prop.target bounds = [] for i in range(len(prop.data)): if prop.data[i] + prop.epsilon > prop.bounds[i][1]: ub = prop.bounds[i][1] else: ub = prop.data[i] + prop.epsilon if prop.data[i] - prop.epsilon < prop.bounds[i][0]: lb = prop.bounds[i][0] else: lb = prop.data[i] - prop.epsilon bounds.append((lb, ub)) else: raise NotImplementedError if not targeted: raise NotImplementedError onnx_rep = cv.ONNXConverter().from_neural_network(network) onnx.save_model(onnx_rep.onnx_network, "temp/onnx_network.onnx") marabou_onnx_net = Marabou.read_onnx("temp/onnx_network.onnx") os.remove("temp/onnx_network.onnx") input_vars = marabou_onnx_net.inputVars[0][0] output_vars = marabou_onnx_net.outputVars assert(len(bounds) == len(input_vars)) for i in range(len(input_vars)): marabou_onnx_net.setLowerBound(input_vars[i], bounds[i][0]) marabou_onnx_net.setUpperBound(input_vars[i], bounds[i][1]) for i in range(len(output_vars)): if i != target: MarabouUtils.addInequality(marabou_onnx_net, [output_vars[i], output_vars[target]], [1, -1], 0) options = MarabouCore.Options() # options._verbosity = 2 vals, stats = marabou_onnx_net.solve(options=options) counterexample = None if not vals: sat = False else: sat = True counterexample = [val for val in vals.values()] counterexample = np.array(counterexample) return sat, counterexample
** in the top-level source directory) and their institutional affiliations. ** All rights reserved. See the file COPYING in the top-level source ** directory for licensing information.\endverbatim ** ** \brief [[ Add one-line brief description here ]] ** ** [[ Add lengthier description here ]] **/ ''' from maraboupy import Marabou from maraboupy import MarabouCore import numpy as np # Set the Marabou option to restrict printing options = MarabouCore.Options() options._verbosity = 0 ### FULLY CONNECTED NETWORK EXAMPLE ### # Network corresponds to inputs x0, x1 # Outputs: y0 = |x0| + |x1|, y1 = x0^2 + x1^2 print("Fully Connected Network Example") filename = './networks/graph_test_medium.onnx' network = Marabou.read_onnx(filename) ## Or, you can specify the operation names of the input and output operations ## By default chooses the only placeholder as input, last op as output #inputName = 'Placeholder:0' #outputName = 'y_out:0' #network = Marabou.read_onnx(filename=filename, inputNames=[inputName], outputName = outputName)