def verify(dnn: OperationGraph, phi: Expression):
logger = logging.getLogger(__name__)
dnn = dnn.simplify()
phi.networks[0].concretize(dnn)
result = UNSAT
property_extractor = ConvexPolytopeExtractor()
with tempfile.TemporaryDirectory() as dirname:
for prop in property_extractor.extract_from(phi):
layers = prop.output_constraint.as_layers(
prop.network, translator_error=ReluplexTranslatorError
)
input_interval = prop.input_constraint.as_hyperrectangle()
nnet_file_name = to_nnet_file(
input_interval,
layers,
dirname=dirname,
translator_error=ReluplexTranslatorError,
)
logger.debug("Running reluplex")
executor = CommandLineExecutor(
"reluplex", f"{nnet_file_name}", verifier_error=ReluplexError
)
out, err = executor.run()
logger.debug("Parsing results")
result |= parse_results(out, err)
if result == SAT:
logger.debug("SAT! Validating counter example.")
validate_counter_example(prop, out, err)
if result == SAT or result == UNKNOWN:
return result
return result
def verify(dnn: OperationGraph, phi: Expression, **kwargs: Dict[str, Any]):
logger = logging.getLogger(__name__)
dnn = dnn.simplify()
phi.networks[0].concretize(dnn)
result = UNSAT
property_extractor = ConvexPolytopeExtractor()
with tempfile.TemporaryDirectory() as dirname:
for prop in property_extractor.extract_from(phi):
layers = prop.output_constraint.as_layers(
prop.network,
extra_layer_types=MIPVERIFY_LAYER_TYPES,
translator_error=MIPVerifyTranslatorError,
)
input_interval = prop.input_constraint.as_hyperrectangle()
mipverify_inputs = to_mipverify_inputs(
input_interval,
layers,
dirname=dirname,
translator_error=MIPVerifyTranslatorError,
)
logger.debug("Running mipverify")
executor = CommandLineExecutor(
"julia",
mipverify_inputs["property_path"],
verifier_error=MIPVerifyError,
)
out, err = executor.run()
logger.debug("Parsing results")
result |= parse_results(out, err)
if result == SAT or result == UNKNOWN:
return result
return result
def extract_from(self, expression: Expression) -> Iterable[Property]:
logger = logging.getLogger(__name__)
self.existential = False
if isinstance(expression, Exists):
raise NotImplementedError() # TODO
expression = ~expression
self.existential = True
expression = expression.canonical()
not_expression = Or(~expression)
for conjunction in not_expression:
logger.info("CONJUNCTION: %s", conjunction)
if len(conjunction.networks) != 1:
continue
raise self.translator_error(
"Exactly one network input and output are required")
if len(conjunction.variables) != 1:
raise self.translator_error(
"Exactly one network input is required")
for prop in super().extract_from(conjunction):
if prop.input_constraint.as_hyperrectangle(
).is_consistent == False:
continue
if prop.output_constraint.is_consistent == False:
continue
yield prop
def verify(dnn: OperationGraph,
phi: Expression,
domain="deeppoly",
timeout_lp=1.0,
timeout_milp=1.0,
use_area_heuristic=True,
**kwargs: Dict[str, Any]):
logger = logging.getLogger(__name__)
dnn = dnn.simplify()
phi.networks[0].concretize(dnn)
result = UNSAT
property_extractor = HalfspacePolytopePropertyExtractor(
HyperRectangle, HalfspacePolytope)
for prop in property_extractor.extract_from(~phi):
if prop.input_constraint.num_variables > 1:
raise ERANTranslatorError(
"Unsupported network: More than 1 input variable")
with tf.Session(graph=tf.Graph()) as tf_session:
layers = as_layers(
prop.suffixed_op_graph(),
extra_layer_types=ERAN_LAYER_TYPES,
translator_error=ERANTranslatorError,
)
input_interval = prop.input_constraint
spec_lb = input_interval.lower_bounds[0]
spec_ub = input_interval.upper_bounds[0]
if len(spec_lb.shape) == 4:
spec_lb = spec_lb.transpose((0, 2, 3, 1))
spec_ub = spec_ub.transpose((0, 2, 3, 1))
tf_graph = as_tf(layers, translator_error=ERANTranslatorError)
eran_model = ERAN(tf_graph, session=tf_session)
_, nn, nlb, nub = eran_model.analyze_box(spec_lb.flatten().copy(),
spec_ub.flatten().copy(),
domain, timeout_lp,
timeout_milp,
use_area_heuristic,
**kwargs)
output_lower_bound = np.asarray(nlb[-1])
output_upper_bound = np.asarray(nub[-1])
logger.debug("output lower bound: %s", output_lower_bound)
logger.debug("output upper bound: %s", output_upper_bound)
result |= check(output_lower_bound, output_upper_bound)
if result == SAT or result == UNKNOWN:
return result
return result
def verify(dnn: OperationGraph, phi: Expression, **kwargs: Dict[str, Any]):
logger = logging.getLogger(__name__)
dnn = dnn.simplify()
phi.networks[0].concretize(dnn)
result = UNSAT
property_extractor = HalfspacePolytopePropertyExtractor(
HyperRectangle, HalfspacePolytope)
with tempfile.TemporaryDirectory() as dirname:
for prop in property_extractor.extract_from(~phi):
if prop.input_constraint.num_variables > 1:
raise NeurifyTranslatorError(
"Unsupported network: More than 1 input variable")
layers = as_layers(
prop.suffixed_op_graph(),
translator_error=NeurifyTranslatorError,
)
neurify_inputs = to_neurify_inputs(
prop.input_constraint,
layers,
dirname=dirname,
translator_error=NeurifyTranslatorError,
)
logger.debug("Running neurify")
executor = CommandLineExecutor(
"neurify",
"-n",
neurify_inputs["nnet_path"],
"-x",
neurify_inputs["input_path"],
"-sl",
"0.0000000000001", # TODO: remove magic number
"-I",
neurify_inputs["input_interval_path"],
"-v",
*[f"--{k}={v}" for k, v in kwargs.items() if v is not None],
verifier_error=NeurifyError,
)
out, err = executor.run()
logger.debug("Parsing results")
result |= parse_results(out, err)
if result == SAT:
logger.debug("SAT! Validating counter example.")
validate_counter_example(prop, out, err)
if result == SAT or result == UNKNOWN:
return result
return result
def run(self):
logger = logging.getLogger(__name__)
arg_string = " ".join(self.args)
logger.info(f"EXECUTING: {arg_string}")
proc = None
try:
proc = sp.Popen(self.args,
stdout=sp.PIPE,
stderr=sp.PIPE,
encoding="utf8")
self.output_lines = []
self.error_lines = []
while proc.poll() is None:
for (name, stream, lines) in [
("STDOUT", proc.stdout, self.output_lines),
("STDERR", proc.stderr, self.error_lines),
]:
ready, _, _ = select.select([stream], [], [], 0)
if not ready:
continue
line = stream.readline()
if not line:
continue
line = line.strip()
lines.append(line)
logger.debug(f"[{name}]:{line}")
for line in proc.stdout.readlines():
line = line.strip()
logger.debug(f"[STDOUT]:{line}")
self.output_lines.append(line)
for line in proc.stderr.readlines():
line = line.strip()
logger.debug(f"[STDERR]:{line}")
self.error_lines.append(line)
if proc.returncode != 0:
raise self.verifier_error(f"Return code: {proc.returncode}")
finally:
if proc is not None:
proc.stderr.close()
proc.stdout.close()
return self.output_lines, self.error_lines
def verify(dnn: OperationGraph, phi: Expression, **kwargs: Dict[str, Any]):
logger = logging.getLogger(__name__)
dnn = dnn.simplify()
phi.networks[0].concretize(dnn)
result = UNSAT
property_extractor = ConvexPolytopeExtractor()
with tempfile.TemporaryDirectory() as dirname:
for prop in property_extractor.extract_from(phi):
layers = prop.output_constraint.as_layers(
prop.network, translator_error=NeurifyTranslatorError)
input_interval = prop.input_constraint.as_hyperrectangle()
neurify_inputs = to_neurify_inputs(
input_interval,
layers,
dirname=dirname,
translator_error=NeurifyTranslatorError,
)
epsilon = neurify_inputs["epsilon"]
logger.debug("Running neurify")
executor = CommandLineExecutor(
"neurify",
"-n",
neurify_inputs["nnet_path"],
"-x",
neurify_inputs["input_path"],
"-sl",
"0.000000000001", # TODO: remove magic number
f"--linf={epsilon}",
"-v",
*[f"--{k}={v}" for k, v in kwargs.items() if v is not None],
verifier_error=NeurifyError,
)
out, err = executor.run()
logger.debug("Parsing results")
result |= parse_results(out, err)
if result == SAT:
logger.debug("SAT! Validating counter example.")
validate_counter_example(prop, out, err)
if result == SAT or result == UNKNOWN:
return result
return result
def run(self):
logger = logging.getLogger(__name__)
arg_string = " ".join(self.args)
logger.info(f"EXECUTING: {arg_string}")
try:
proc = sp.Popen(self.args, stdout=sp.PIPE, stderr=sp.PIPE, encoding="utf8")
self.output_lines = []
self.error_lines = []
while proc.poll() is None:
readables, _, _ = select.select([proc.stdout, proc.stderr], [], [], 0)
while len(readables) > 0:
io = readables.pop()
line = io.readline().strip()
if not line:
continue
prefix = ""
if io.fileno() == proc.stdout.fileno():
self.output_lines.append(line)
prefix = "[STDOUT]:"
elif io.fileno() == proc.stderr.fileno():
self.error_lines.append(line)
prefix = "[STDERR]:"
logger.debug(f"{prefix}{line}")
if proc.returncode != 0:
raise self.verifier_error(f"Received signal: {-proc.returncode}")
for line in proc.stderr.readlines():
line = line.strip()
if line:
logger.debug(f"[STDERR]:{line}")
self.error_lines.append(line)
for line in proc.stdout.readlines():
line = line.strip()
if line:
logger.debug(f"[STDOUT]:{line}")
self.output_lines.append(line)
finally:
proc.stderr.close()
proc.stdout.close()
return self.output_lines, self.error_lines
def verify(dnn: OperationGraph, phi: Expression, **kwargs: Dict[str, Any]):
logger = logging.getLogger(__name__)
dnn = dnn.simplify()
phi.networks[0].concretize(dnn)
result = UNSAT
property_extractor = HalfspacePolytopePropertyExtractor(
HyperRectangle, HalfspacePolytope)
with tempfile.TemporaryDirectory() as dirname:
for prop in property_extractor.extract_from(~phi):
if prop.input_constraint.num_variables > 1:
raise MIPVerifyTranslatorError(
"Unsupported network: More than 1 input variable")
layers = as_layers(
prop.suffixed_op_graph(),
extra_layer_types=MIPVERIFY_LAYER_TYPES,
translator_error=MIPVerifyTranslatorError,
)
mipverify_inputs = to_mipverify_inputs(
prop.input_constraint,
layers,
dirname=dirname,
translator_error=MIPVerifyTranslatorError,
)
logger.debug("Running mipverify")
executor = CommandLineExecutor(
"julia",
mipverify_inputs["property_path"],
verifier_error=MIPVerifyError,
)
out, err = executor.run()
logger.debug("Parsing results")
result |= parse_results(out, err)
if result == SAT or result == UNKNOWN:
return result
return result