def evaluateWithMarabou(self,
inputValues,
filename="evaluateWithMarabou.log",
options=None):
"""
Function to evaluate network at a given point using Marabou as solver
Arguments:
inputValues: list of (np arrays) representing input to network
filename: (string) path to redirect output
Returns:
outputValues: (np array) representing output of network
"""
inputVars = self.inputVars # list of numpy arrays
outputVars = self.outputVars
inputDict = dict()
inputVarList = np.concatenate(inputVars, axis=-1).ravel()
inputValList = np.concatenate(inputValues).ravel()
assignList = zip(inputVarList, inputValList)
for x in assignList:
inputDict[x[0]] = x[1]
ipq = self.getMarabouQuery()
for k in inputDict:
ipq.setLowerBound(k, inputDict[k])
ipq.setUpperBound(k, inputDict[k])
if options == None:
options = MarabouCore.Options()
outputDict = MarabouCore.solve(ipq, options, filename)
outputValues = outputVars.reshape(-1).astype(np.float64)
for i in range(len(outputValues)):
outputValues[i] = (outputDict[0])[outputValues[i]]
outputValues = outputValues.reshape(outputVars.shape)
return outputValues
def solve(self, filename="", verbose=True, options=None):
"""Function to solve query represented by this network
Args:
filename (string): Path for redirecting output
verbose (bool): If true, print out solution after solve finishes
options (:class:`~maraboupy.MarabouCore.Options`): Object for specifying Marabou options, defaults to None
Returns:
(tuple): tuple containing:
- exitCode (str): A string representing the exit code (sat/unsat/TIMEOUT/ERROR/UNKNOWN/QUIT_REQUESTED).
- vals (Dict[int, float]): Empty dictionary if UNSAT, otherwise a dictionary of SATisfying values for variables
- stats (:class:`~maraboupy.MarabouCore.Statistics`): A Statistics object to how Marabou performed
"""
ipq = self.getMarabouQuery()
if options == None:
options = MarabouCore.Options()
exitCode, vals, stats = MarabouCore.solve(ipq, options, str(filename))
if verbose:
print(exitCode)
if exitCode == "sat":
for j in range(len(self.inputVars)):
for i in range(self.inputVars[j].size):
print("input {} = {}".format(
i, vals[self.inputVars[j].item(i)]))
for j in range(len(self.outputVars)):
for i in range(self.outputVars[j].size):
print("output {} = {}".format(
i, vals[self.outputVars[j].item(i)]))
return [exitCode, vals, stats]
def solve(self, filename="", verbose=True, options=None):
"""
Function to solve query represented by this network
Arguments:
filename: (string) path to redirect output to
verbose: (bool) whether to print out solution after solve finishes
options: (MarabouCore.Options) object for specifying Marabou options
Returns:
vals: (dict: int->float) empty if UNSAT, else SATisfying solution
stats: (Statistics) a Statistics object as defined in Marabou,
it has multiple methods that provide information related
to how an input query was solved.
"""
ipq = self.getMarabouQuery()
if options == None:
options = MarabouCore.Options()
vals, stats = MarabouCore.solve(ipq, options, filename)
if verbose:
if stats.hasTimedOut():
print("TO")
elif len(vals) == 0:
print("unsat")
else:
print("sat")
for j in range(len(self.inputVars)):
for i in range(self.inputVars[j].size):
print("input {} = {}".format(
i, vals[self.inputVars[j].item(i)]))
for i in range(self.outputVars.size):
print("output {} = {}".format(
i, vals[self.outputVars.item(i)]))
return [vals, stats]
def solve(self, filename="", verbose=True, options=None):
"""Function to solve query represented by this network
Args:
filename (string): Path for redirecting output
verbose (bool): If true, print out solution after solve finishes
options (:class:`~maraboupy.MarabouCore.Options`): Object for specifying Marabou options, defaults to None
Returns:
(tuple): tuple containing:
- vals (Dict[int, float]): Empty dictionary if UNSAT, otherwise a dictionary of SATisfying values for variables
- stats (:class:`~maraboupy.MarabouCore.Statistics`): A Statistics object to how Marabou performed
"""
ipq = self.getMarabouQuery()
if options == None:
options = MarabouCore.Options()
vals, stats = MarabouCore.solve(ipq, options, filename)
if verbose:
if stats.hasTimedOut():
print("TO")
elif len(vals) == 0:
print("unsat")
else:
print("sat")
for j in range(len(self.inputVars)):
for i in range(self.inputVars[j].size):
print("input {} = {}".format(
i, vals[self.inputVars[j].item(i)]))
for i in range(self.outputVars.size):
print("output {} = {}".format(
i, vals[self.outputVars.item(i)]))
return [vals, stats]
def evaluateWithMarabou(self,
inputValues,
filename="evaluateWithMarabou.log",
options=None):
"""Function to evaluate network at a given point using Marabou as solver
Args:
inputValues (list of np arrays): Inputs to evaluate
filename (str): Path to redirect output if using Marabou solver, defaults to "evaluateWithMarabou.log"
options (:class:`~maraboupy.MarabouCore.Options`): Object for specifying Marabou options, defaults to None
Returns:
(list of np arrays): Values representing the outputs of the network or None if system is UNSAT
"""
# Make sure inputValues is a list of np arrays and not list of lists
inputValues = [np.array(inVal) for inVal in inputValues]
inputVars = self.inputVars # list of numpy arrays
outputVars = self.outputVars # list of numpy arrays
inputDict = dict()
inputVarList = np.concatenate([inVar.flatten() for inVar in inputVars],
axis=-1).flatten()
inputValList = np.concatenate(
[inVal.flatten() for inVal in inputValues]).flatten()
assignList = zip(inputVarList, inputValList)
for x in assignList:
inputDict[x[0]] = x[1]
ipq = self.getMarabouQuery()
for k in inputDict:
ipq.setLowerBound(k, inputDict[k])
ipq.setUpperBound(k, inputDict[k])
if options == None:
options = MarabouCore.Options()
exitCode, outputDict, _ = MarabouCore.solve(ipq, options,
str(filename))
# When the query is UNSAT an empty dictionary is returned
if outputDict == {}:
return None
outputValues = [
outVars.reshape(-1).astype(np.float64) for outVars in outputVars
]
for i in range(len(outputValues)):
for j in range(len(outputValues[i])):
outputValues[i][j] = outputDict[outputValues[i][j]]
outputValues[i] = outputValues[i].reshape(outputVars[i].shape)
return outputValues
def solve(self, filename="", verbose=True, options=None):
"""
Function to solve query represented by this network
Arguments:
filename: (string) path to redirect output to
verbose: (bool) whether to print out solution after solve finishes
timeout: (int) time in seconds when Marabou will time out
verbosity: (int) determines how much Marabou prints during solving
0: print out minimal information
1: print out statistics only in the beginning and the end
2: print out statistics during solving
Returns:
vals: (dict: int->float) empty if UNSAT, else SATisfying solution
stats: (Statistics) a Statistics object as defined in Marabou,
it has multiple methods that provide information related
to how an input query was solved.
"""
ipq = self.getMarabouQuery()
if options == None:
options = MarabouCore.Options()
vals, stats = MarabouCore.solve(ipq, options, filename)
if verbose:
if stats.hasTimedOut():
print("TO")
elif len(vals) == 0:
print("UNSAT")
else:
print("SAT")
for inputVarArray in self.inputVars:
for inputVar in inputVarArray.flatten():
print("input {} = {}".format(inputVar, vals[inputVar]))
# print("input var {} input {} = {}".format(i, self.inputVars[j][0][i],vals[self.inputVars[j].item(i)]))
# for j in range(len(self.inputVars)):
# for i in range(self.inputVars[j].size):
# print("input {} = {}".format(i, vals[self.inputVars[j].item(i)]))
# print("input var {} input {} = {}".format(i, self.inputVars[j][0][i],vals[self.inputVars[j].item(i)]))
for i in range(self.outputVars.size):
print("output {} = {}".format(
i, vals[self.outputVars.item(i)]))
return [vals, stats]
def evaluateWithMarabou(self,
inputValues,
filename="evaluateWithMarabou.log",
options=None):
"""
Function to evaluate network at a given point using Marabou as solver
Arguments:
inputValues: list of (np arrays) representing input to network
filename: (string) path to redirect output
options: (MarabouCore.Options) object for specifying Marabou options
Returns:
outputValues: (np array) representing output of network
"""
# Make sure inputValues is a list of np arrays and not list of lists
inputValues = [np.array(inVal) for inVal in inputValues]
inputVars = self.inputVars # list of numpy arrays
outputVars = self.outputVars
inputDict = dict()
inputVarList = np.concatenate([inVar.flatten() for inVar in inputVars],
axis=-1).flatten()
inputValList = np.concatenate(
[inVal.flatten() for inVal in inputValues]).flatten()
assignList = zip(inputVarList, inputValList)
for x in assignList:
inputDict[x[0]] = x[1]
ipq = self.getMarabouQuery()
for k in inputDict:
ipq.setLowerBound(k, inputDict[k])
ipq.setUpperBound(k, inputDict[k])
if options == None:
options = MarabouCore.Options()
outputDict, _ = MarabouCore.solve(ipq, options, filename)
outputValues = outputVars.reshape(-1).astype(np.float64)
for i in range(len(outputValues)):
outputValues[i] = outputDict[outputValues[i]]
outputValues = outputValues.reshape(outputVars.shape)
return outputValues
def evaluateLocalRobustness(self,
input,
epsilon,
originalClass,
verbose=True,
options=None,
targetClass=None):
"""Function evaluating a specific input is a local robustness within the scope of epslion
Args:
input (numpy.ndarray): Target input
epsilon (float): L-inf norm of purturbation
originalClass (int): Output class of a target input
verbose (bool): If true, print out solution after solve finishes
options (:class:`~maraboupy.MarabouCore.Options`): Object for specifying Marabou options, defaults to None
targetClass (int): If set, find a feasible solution with which the value of targetClass is max within outputs.
Returns:
(tuple): tuple containing:
- vals (Dict[int, float]): Empty dictionary if UNSAT, otherwise a dictionary of SATisfying values for variables
- stats (:class:`~maraboupy.MarabouCore.Statistics`): A Statistics object to how Marabou performed
- maxClass (int): Output class which value is max within outputs if SAT.
"""
inputVars = None
if (type(self.inputVars) is list):
if (len(self.inputVars) != 1):
raise NotImplementedError(
"Operation for %d inputs is not implemented" %
len(self.inputVars))
inputVars = self.inputVars[0][0]
elif (type(self.inputVars) is np.ndarray):
inputVars = self.inputVars[0]
else:
err_msg = "Unpexpected type of input vars."
raise RuntimeError(err_msg)
if inputVars.shape != input.shape:
raise RuntimeError(
"Input shape of the model should be same as the input shape\n input shape of the model: {0}, shape of the input: {1}"
.format(inputVars.shape, input.shape))
if (type(self.outputVars) is list):
if (len(self.outputVars) != 1):
raise NotImplementedError(
"Operation for %d outputs is not implemented" %
len(self.outputVars))
elif (type(self.outputVars) is np.ndarray):
if (len(self.outputVars) != 1):
raise NotImplementedError(
"Operation for %d outputs is not implemented" %
len(self.outputVars))
else:
err_msg = "Unpexpected type of output vars."
raise RuntimeError(err_msg)
if options == None:
options = MarabouCore.Options()
# Add constratins to all input nodes
flattenInputVars = inputVars.flatten()
flattenInput = input.flatten()
for i in range(flattenInput.size):
self.setLowerBound(flattenInputVars[i], flattenInput[i] - epsilon)
self.setUpperBound(flattenInputVars[i], flattenInput[i] + epsilon)
maxClass = None
outputStartIndex = self.outputVars[0][0][0]
if targetClass is None:
outputLayerSize = len(self.outputVars[0][0])
# loop for all of output classes except for original class
for outputLayerIndex in range(outputLayerSize):
if outputLayerIndex != originalClass:
self.addMaxConstraint(
set([
outputStartIndex + outputLayerIndex,
outputStartIndex + originalClass
]), outputStartIndex + outputLayerIndex)
exitCode, vals, stats = self.solve(options=options)
if (stats.hasTimedOut()):
break
elif (len(vals) > 0):
maxClass = outputLayerIndex
break
else:
self.addMaxConstraint(set(self.outputVars[0][0]),
outputStartIndex + targetClass)
exitCode, vals, stats = self.solve(options=options)
if verbose:
if not stats.hasTimedOut() and len(vals) > 0:
maxClass = targetClass
# print timeout, or feasible inputs and outputs if verbose is on.
if verbose:
if stats.hasTimedOut():
print("TO")
elif len(vals) > 0:
print("sat")
for j in range(len(self.inputVars[0])):
for i in range(self.inputVars[0][j].size):
print("input {} = {}".format(
i, vals[self.inputVars[0][j].item(i)]))
for j in range(len(self.outputVars[0])):
for i in range(self.outputVars[0][j].size):
print("output {} = {}".format(
i, vals[self.outputVars[0][j].item(i)]))
return [vals, stats, maxClass]
def verify(self, network: networks.NeuralNetwork, prop: Property) -> (bool, typing.Optional[Tensor.Tensor]):
"""
Verify that the neural network of interest satisfy the property given as argument
using the Marabou verification tool.
Parameters
----------
network : NeuralNetwork
The neural network to train.
prop : Dataset
The property which the neural network must satisfy.
Returns
----------
(bool, Optional[Tensor])
True and None if the neural network satisfy the property, False and the counterexample otherwise.
"""
if isinstance(prop, SMTLIBProperty):
targeted, bounds, target = utilities.parse_linf_robustness_smtlib(prop.smtlib_path)
elif isinstance(prop, LocalRobustnessProperty):
targeted = prop.targeted
target = prop.target
bounds = []
for i in range(len(prop.data)):
if prop.data[i] + prop.epsilon > prop.bounds[i][1]:
ub = prop.bounds[i][1]
else:
ub = prop.data[i] + prop.epsilon
if prop.data[i] - prop.epsilon < prop.bounds[i][0]:
lb = prop.bounds[i][0]
else:
lb = prop.data[i] - prop.epsilon
bounds.append((lb, ub))
else:
raise NotImplementedError
if not targeted:
raise NotImplementedError
onnx_rep = cv.ONNXConverter().from_neural_network(network)
onnx.save_model(onnx_rep.onnx_network, "temp/onnx_network.onnx")
marabou_onnx_net = Marabou.read_onnx("temp/onnx_network.onnx")
os.remove("temp/onnx_network.onnx")
input_vars = marabou_onnx_net.inputVars[0][0]
output_vars = marabou_onnx_net.outputVars
assert(len(bounds) == len(input_vars))
for i in range(len(input_vars)):
marabou_onnx_net.setLowerBound(input_vars[i], bounds[i][0])
marabou_onnx_net.setUpperBound(input_vars[i], bounds[i][1])
for i in range(len(output_vars)):
if i != target:
MarabouUtils.addInequality(marabou_onnx_net, [output_vars[i], output_vars[target]], [1, -1], 0)
options = MarabouCore.Options()
# options._verbosity = 2
vals, stats = marabou_onnx_net.solve(options=options)
counterexample = None
if not vals:
sat = False
else:
sat = True
counterexample = [val for val in vals.values()]
counterexample = np.array(counterexample)
return sat, counterexample
** in the top-level source directory) and their institutional affiliations.
** All rights reserved. See the file COPYING in the top-level source
** directory for licensing information.\endverbatim
**
** \brief [[ Add one-line brief description here ]]
**
** [[ Add lengthier description here ]]
**/
'''
from maraboupy import Marabou
from maraboupy import MarabouCore
import numpy as np
# Set the Marabou option to restrict printing
options = MarabouCore.Options()
options._verbosity = 0
### FULLY CONNECTED NETWORK EXAMPLE ###
# Network corresponds to inputs x0, x1
# Outputs: y0 = |x0| + |x1|, y1 = x0^2 + x1^2
print("Fully Connected Network Example")
filename = './networks/graph_test_medium.onnx'
network = Marabou.read_onnx(filename)
## Or, you can specify the operation names of the input and output operations
## By default chooses the only placeholder as input, last op as output
#inputName = 'Placeholder:0'
#outputName = 'y_out:0'
#network = Marabou.read_onnx(filename=filename, inputNames=[inputName], outputName = outputName)
