def verify_without_ar(net, json_content, callback=None, filename=""):
print("query using vanilla Marabou")
print(json_content)
parameter = json.loads(json_content)
test_property = generate_test_property(parameter)
net = copy.deepcopy(net)
# test_property = get_test_property_acas(property_id)
dynamically_import_marabou(query_type=test_property["type"])
net, test_property = reduce_property_to_basic_form(
network=net, test_property=test_property)
t0 = time.time()
vars1, stats1, query_result = get_query(network=net,
test_property=test_property,
verbose=consts.VERBOSE)
t1 = time.time()
# time to check property on net with marabou
marabou_time = t1 - t0
print(f"query time = {marabou_time}")
res = {
"query_result": query_result,
"orig_query_time": str(marabou_time),
"net_data": str(net.get_general_net_data()),
}
return json.dumps(res)
def one_experiment(nnet_filename, is_tiny, refinement_type, abstraction_type,
epsilon, lower_bound, preprocess_orig_net,
refinement_sequence_length, abstraction_sequence_length,
results_directory, property_id=consts.PROPERTY_ID,
verbose=consts.VERBOSE):
if verbose:
debug_print("one_experiment_cegarabou({})".format(json.dumps([nnet_filename, is_tiny, refinement_type,
abstraction_type, epsilon, lower_bound,
property_id, preprocess_orig_net])))
if is_tiny:
example_nets_dir_path = consts.PATH_TO_MARABOU_ACAS_EXAMPLES
else:
example_nets_dir_path = consts.PATH_TO_MARABOU_APPLICATIONS_ACAS_EXAMPLES
fullname = os.path.join(example_nets_dir_path, nnet_filename)
if not os.path.exists(results_directory):
os.makedirs(results_directory)
results_filename = generate_results_filename(nnet_filename=nnet_filename,
is_tiny=is_tiny,
refinement_type=refinement_type,
abstraction_type=abstraction_type,
epsilon=epsilon,
lower_bound=lower_bound,
property_id=property_id,
preprocess_orig_net=preprocess_orig_net,
abstraction_sequence_length=abstraction_sequence_length,
refinement_sequence_length=refinement_sequence_length)
test_property = get_test_property_tiny() if is_tiny else get_test_property_acas(property_id)
# for i in range(len(test_property["output"])):
# test_property["output"][i][1]["Lower"] = lower_bound
net = network_from_nnet_file(fullname)
print(f"size={len(net.layers)}")
orig_net = copy.deepcopy(net)
if args.preprocess_orig_net:
preprocess(orig_net)
if verbose:
print("query using AR")
t2 = time.time()
if abstraction_type == "complete":
print("complete")
net = abstract_network(net)
else:
print("heuristic")
net = heuristic_abstract(network=net, test_property=test_property,
sequence_length=abstraction_sequence_length)
abstraction_time = time.time() - t2
num_of_refine_steps = 0
ar_times = []
ar_sizes = []
refine_sequence_times = []
while True: # CEGAR / CETAR method
t4 = time.time()
vars1, stats1, query_result = get_query(
network=net, test_property=test_property,
verbose=consts.VERBOSE
)
debug_print(f'query_result={query_result}')
t5 = time.time()
ar_times.append(t5 - t4)
ar_sizes.append(net.get_general_net_data()["num_nodes"])
if verbose:
print("query time after A and {} R steps is {}".format(num_of_refine_steps, t5-t4))
debug_print(net.get_general_net_data())
if query_result == "UNSAT":
# if always y'<3.99 then also always y<3.99
if verbose:
print("UNSAT (finish)")
break
if query_result == "SAT":
if verbose:
print("SAT (have to check example on original net)")
print(vars1)
debug_print(f'vars1={vars1}')
st = time.time()
orig_net_output = orig_net.evaluate(vars1)
print("evaluate: {}".format(time.time() - st))
st = time.time()
orig_net.speedy_evaluate(vars1)
print("speedy evaluate: {}".format(time.time() - st))
nodes2variables, variables2nodes = orig_net.get_variables()
# we got y'>3.99, check if also y'>3.99 for the same input
if is_satisfying_assignment(network=orig_net,
test_property=test_property,
output=orig_net_output,
variables2nodes=variables2nodes):
if verbose:
print("property holds also in orig - SAT (finish)")
break # also counter example for orig_net
else:
t_cur_refine_start = time.time()
if verbose:
print("property doesn't holds in orig - spurious example")
num_of_refine_steps += 1
if verbose:
print("refine step #{}".format(num_of_refine_steps))
if refinement_type == "cegar":
debug_print("cegar")
net = refine(network=net,
sequence_length=refinement_sequence_length,
example=vars1)
else:
debug_print("cetar")
net = refine(network=net,
sequence_length=refinement_sequence_length)
t_cur_refine_end = time.time()
refine_sequence_times.append(t_cur_refine_end - t_cur_refine_start)
t3 = time.time()
# time to check property on net with marabou using CEGAR
total_ar_time = t3 - t2
if verbose:
print("ar query time = {}".format(total_ar_time))
# time to check property on the last network in CEGAR
last_net_ar_time = t3 - t4
if verbose:
print("last ar net query time = {}".format(last_net_ar_time))
res = [
("net name", nnet_filename),
("property_id", property_id),
("abstraction_time", abstraction_time),
("query_result", query_result),
("num_of_refine_steps", num_of_refine_steps),
("total_ar_query_time", total_ar_time),
("ar_times", json.dumps(ar_times)),
("ar_sizes", json.dumps(ar_sizes)),
("refine_sequence_times", json.dumps(refine_sequence_times)),
("last_net_data", json.dumps(net.get_general_net_data())),
("last_query_time", last_net_ar_time)
]
# generate dataframe from result
df = pd.DataFrame.from_dict({x[0]: [x[1]] for x in res})
df.to_json(os.path.join(results_directory, "df_" + results_filename))
# write result to output file
with open(os.path.join(results_directory, results_filename), "w") as fw:
fw.write("-"*80)
fw.write("parameters:")
fw.write("-"*80)
fw.write("\n")
for arg in vars(args):
fw.write("{}: {}\n".format(arg, getattr(args, arg)))
fw.write("+"*80)
fw.write("results:")
fw.write("+"*80)
fw.write("\n")
for (k,v) in res:
fw.write("{}: {}\n".format(k,v))
return res
def one_experiment(nnet_filename,
refinement_type,
abstraction_type,
mechanism,
refinement_sequence,
abstraction_sequence,
results_directory,
property_id=consts.PROPERTY_ID,
verbose=consts.VERBOSE):
test_property = get_test_property_acas(property_id)
dynamically_import_marabou(query_type=test_property["type"])
from core.nnet.read_nnet import (network_from_nnet_file, network2rlv)
from core.abstraction.naive import abstract_network
from core.abstraction.alg2 import heuristic_abstract_alg2
from core.abstraction.random_abstract import heuristic_abstract_random
# from core.abstraction.clustering_abstract import \
# heuristic_abstract_clustering
from core.utils.marabou_query_utils import reduce_property_to_basic_form, get_query
from core.refinement.refine import refine
example_nets_dir_path = consts.PATH_TO_MARABOU_APPLICATIONS_ACAS_EXAMPLES
fullname = os.path.join(example_nets_dir_path, nnet_filename)
if not os.path.exists(results_directory):
os.makedirs(results_directory)
results_filename = generate_results_filename(
nnet_filename=nnet_filename,
property_id=property_id,
mechanism=mechanism,
refinement_type=refinement_type,
abstraction_type=abstraction_type,
refinement_sequence=refinement_sequence,
abstraction_sequence=abstraction_sequence)
# for i in range(len(test_property["output"])):
# test_property["output"][i][1]["Lower"] = lower_bound
net = network_from_nnet_file(fullname)
# print(net)
print(f"size={len(net.layers)}")
net, test_property = reduce_property_to_basic_form(
network=net, test_property=test_property)
# mechanism is vanilla marabou
if mechanism == "marabou":
print("query using vanilla Marabou")
t0 = time.time()
vars1, stats1, query_result = get_query(network=net,
test_property=test_property,
verbose=consts.VERBOSE)
t1 = time.time()
# time to check property on net with marabou
marabou_time = t1 - t0
if verbose:
print(f"query time = {marabou_time}")
res = [
("net_name", nnet_filename),
("property_id", property_id),
("query_result", query_result),
("orig_query_time", marabou_time),
("net_data", json.dumps(net.get_general_net_data())),
]
saveResultToFile(results_directory, results_filename, res)
return res
else:
# mechanism is marabou_with_ar
orig_net = copy.deepcopy(net)
print("query using Marabou with AR")
t2 = time.time()
if abstraction_type == "complete":
net = abstract_network(net)
elif abstraction_type == "heuristic_alg2":
net = heuristic_abstract_alg2(network=net,
test_property=test_property,
sequence_length=abstraction_sequence)
elif abstraction_type == "heuristic_random":
net = heuristic_abstract_random(
network=net,
test_property=test_property,
sequence_length=abstraction_sequence)
# elif abstraction_type == "heuristic_clustering":
# net = heuristic_abstract_clustering(
# network=net,
# test_property=test_property,
# sequence_length=abstraction_sequence
# )
else:
raise NotImplementedError("unknown abstraction")
abstraction_time = time.time() - t2
num_of_refine_steps = 0
ar_times = []
ar_sizes = []
refine_sequence_times = []
spurious_examples = []
while True: # CEGAR / CETAR method
t4 = time.time()
print(net.get_general_net_data())
vars1, stats1, query_result = get_query(
network=net,
test_property=test_property,
verbose=consts.VERBOSE)
t5 = time.time()
ar_times.append(t5 - t4)
ar_sizes.append(net.get_general_net_data()["num_nodes"])
# if verbose:
print("query time after A and {} R steps is {}".format(
num_of_refine_steps, t5 - t4))
debug_print(net.get_general_net_data())
if query_result == "UNSAT":
print("+" * 100)
# if always y'<3.99 then also always y<3.99
if verbose:
print("UNSAT (finish)")
break
if query_result == "SAT":
if verbose:
print("SAT (have to check example on original net)")
print(vars1)
# debug_print(f'vars1={vars1}')
# st = time.time()
# orig_net_output = orig_net.evaluate(vars1)
# print("evaluate: {}".format(time.time() - st))
# st = time.time()
orig_net_output = orig_net.speedy_evaluate(vars1)
# print(f"orig_net_output={orig_net_output}")
# print(f"orig_net.name2node_map={orig_net.name2node_map}")
# print("speedy evaluate: {}".format(time.time() - st))
nodes2variables, variables2nodes = orig_net.get_variables()
# we got y'>3.99, check if also y'>3.99 for the same input
if is_satisfying_assignment(network=orig_net,
test_property=test_property,
output=orig_net_output,
variables2nodes=variables2nodes):
if verbose:
print("property holds also in orig - SAT (finish)")
break # also counter example for orig_net
else:
spurious_examples.append(vars1)
t_cur_refine_start = time.time()
if verbose:
print(
"property doesn't holds in orig - spurious example"
)
num_of_refine_steps += 1
if verbose:
print("refine step #{}".format(num_of_refine_steps))
# refine until all spurious examples are satisfied
# since all spurious examples are satisfied in the original
# network, the loop stops until net will be fully refined
refinement_sequences_counter = 0
while True:
refinement_sequences_counter += 1
# print(f"refinement_sequences_counter={refinement_sequences_counter}")
if refinement_type == "cegar":
debug_print("cegar")
net = refine(network=net,
sequence_length=refinement_sequence,
example=vars1)
else:
debug_print("weight_based")
net = refine(network=net,
sequence_length=refinement_sequence)
# after refining, check if the current spurious example is
# already not a counter example (i.e. not satisfied in the
# refined network). stop if not satisfied, continue if yes
net_output = net.speedy_evaluate(vars1)
# print(f"net_output={net_output}")
# print(f"net.name2node_map={net.name2node_map}")
nodes2variables, variables2nodes = net.get_variables()
if not is_satisfying_assignment(
network=net,
test_property=test_property,
output=net_output,
variables2nodes=variables2nodes):
break
t_cur_refine_end = time.time()
refine_sequence_times.append(t_cur_refine_end -
t_cur_refine_start)
t3 = time.time()
# time to check property on net with marabou using CEGAR
total_ar_time = t3 - t2
if verbose:
print("ar query time = {}".format(total_ar_time))
# time to check property on the last network in CEGAR
last_net_ar_time = t3 - t4
if verbose:
print("last ar net query time = {}".format(last_net_ar_time))
res = [("net_name", nnet_filename), ("property_id", property_id),
("abstraction_time", abstraction_time),
("query_result", query_result),
("num_of_refine_steps", num_of_refine_steps),
("total_ar_query_time", total_ar_time),
("ar_times", json.dumps(ar_times)),
("ar_sizes", json.dumps(ar_sizes)),
("refine_sequence_times", json.dumps(refine_sequence_times)),
("last_net_data", json.dumps(net.get_general_net_data())),
("last_query_time", last_net_ar_time)]
return res
def one_experiment(nnet_filename,
is_tiny,
lower_bound,
preprocess_orig_net,
results_directory,
property_id=consts.PROPERTY_ID,
is_adversarial_property=False,
verbose=consts.VERBOSE):
if verbose:
debug_print("one_experiment_marabou({})".format(
json.dumps([
nnet_filename, is_tiny, lower_bound, preprocess_orig_net,
property_id, results_directory
])))
if is_tiny:
example_nets_dir_path = consts.PATH_TO_MARABOU_ACAS_EXAMPLES
else:
example_nets_dir_path = consts.PATH_TO_MARABOU_APPLICATIONS_ACAS_EXAMPLES
fullname = os.path.join(example_nets_dir_path, nnet_filename)
if not os.path.exists(results_directory):
os.makedirs(results_directory)
results_filename = generate_results_filename(
nnet_filename=nnet_filename,
is_tiny=is_tiny,
lower_bound=lower_bound,
preprocess_orig_net=preprocess_orig_net,
property_id=property_id,
is_adversarial_property=is_adversarial_property)
test_property = get_test_property_tiny(
) if is_tiny else get_test_property_acas(property_id)
# for i in range(len(test_property["output"])):
# test_property["output"][i][1]["Lower"] = lower_bound
net = network_from_nnet_file(fullname)
orig_net = copy.deepcopy(net)
if args.preprocess_orig_net:
preprocess(orig_net)
# query original net
if verbose:
print("query orig_net")
t0 = time.time()
if verbose:
debug_print("orig_net.get_general_net_data(): {}".format(
orig_net.get_general_net_data()))
vars1, stats1, query_result = get_query(
network=orig_net,
test_property=test_property,
is_adversarial_property=consts.IS_ADVERSARIAL,
verbose=consts.VERBOSE)
t1 = time.time()
# time to check property on net with marabou
marabou_time = t1 - t0
if verbose:
print("orig_net query time ={}".format(marabou_time))
res = [
("net name", nnet_filename),
("property_id", property_id),
("query_result", query_result),
("orig_query_time", marabou_time),
("net_data", json.dumps(orig_net.get_general_net_data())),
]
# generate dataframe from result
df = pd.DataFrame.from_dict({x[0]: [x[1]] for x in res})
df.to_json(os.path.join(results_directory, "df_" + results_filename))
with open(os.path.join(results_directory, results_filename), "w") as fw:
fw.write("-" * 80)
fw.write("parameters:")
fw.write("-" * 80)
fw.write("\n")
for arg in vars(args):
fw.write("{}: {}\n".format(arg, getattr(args, arg)))
fw.write("+" * 80)
fw.write("results:")
fw.write("+" * 80)
fw.write("\n")
for (k, v) in res:
fw.write("{}: {}\n".format(k, v))
return res
def verify_with_ar(abstract_net,
orig_net,
test_property,
refinement_type,
abstraction_type,
refinement_sequence,
abstraction_sequence,
verbose=consts.VERBOSE):
try:
# mechanism is marabou_with_ar
dynamically_import_marabou(query_type=test_property["type"])
net = copy.deepcopy(abstract_net)
orig_net = copy.deepcopy(orig_net)
test_property = copy.deepcopy(test_property)
num_of_refine_steps = 0
ar_times = []
ar_sizes = []
refine_sequence_times = []
spurious_examples = []
start = time.time()
while True: # CEGAR / CETAR method
t4 = time.time()
vars1, stats1, query_result = get_query(
network=net,
test_property=test_property,
verbose=consts.VERBOSE)
t5 = time.time()
ar_times.append(t5 - t4)
ar_sizes.append(net.get_general_net_data()["num_nodes"])
# if verbose:
print("query time after A and {} R steps is {}".format(
num_of_refine_steps, t5 - t4))
if query_result == "UNSAT":
# if always y'<3.99 then also always y<3.99
if verbose:
print("UNSAT (finish)")
break
if query_result == "SAT":
if verbose:
print("SAT (have to check example on original net)")
orig_net_output = orig_net.speedy_evaluate(vars1)
nodes2variables, variables2nodes = orig_net.get_variables()
# we got y'>3.99, check if also y'>3.99 for the same input
if is_satisfying_assignment(network=orig_net,
test_property=test_property,
output=orig_net_output,
variables2nodes=variables2nodes):
if verbose:
print("property holds also in orig - SAT (finish)")
break # also counter example for orig_net
else:
spurious_examples.append(vars1)
t_cur_refine_start = time.time()
if verbose:
print(
"property doesn't holds in orig - spurious example"
)
num_of_refine_steps += 1
if verbose:
print("refine step #{}".format(num_of_refine_steps))
# refine until all spurious examples are satisfied
# since all spurious examples are satisfied in the original
# network, the loop stops until net will be fully refined
refinement_sequences_counter = 0
while True:
refinement_sequences_counter += 1
# print(f"refinement_sequences_counter={refinement_sequences_counter}")
if refinement_type == "cegar":
debug_print("cegar")
net = refine(network=net,
sequence_length=refinement_sequence,
example=vars1)
else:
debug_print("weight_based")
net = refine(network=net,
sequence_length=refinement_sequence)
# after refining, check if the current spurious example is
# already not a counter example (i.e. not satisfied in the
# refined network). stop if not satisfied, continue if yes
net_output = net.speedy_evaluate(vars1)
# print(f"net_output={net_output}")
# print(f"net.name2node_map={net.name2node_map}")
nodes2variables, variables2nodes = net.get_variables()
if not is_satisfying_assignment(
network=net,
test_property=test_property,
output=net_output,
variables2nodes=variables2nodes):
break
t_cur_refine_end = time.time()
refine_sequence_times.append(t_cur_refine_end -
t_cur_refine_start)
t3 = time.time()
consume = t3 - start
# time to check property on the last network in CEGAR
last_net_ar_time = t3 - t4
if verbose:
print("last ar net query time = {}".format(last_net_ar_time))
res = {
"query_result": query_result,
"num_of_refine_steps": str(num_of_refine_steps),
"ar_times": str(ar_times),
"ar_sizes": str(ar_sizes),
"refine_sequence_times": str(refine_sequence_times),
"refinement_consume": str(consume)
}
return json.dumps(res)
except Exception as e:
print("exception occur", e)