def adjust_layer_after_split_pos_neg(network: Network, layer_index: int = FIRST_POS_NEG_LAYER ) -> None: # debug_print("adjust_layer_after_split_pos_neg") cur_layer = network.layers[layer_index] next_layer = network.layers[layer_index + 1] for node in cur_layer.nodes: node.new_out_edges = [] for next_node in next_layer.nodes: next_node.new_in_edges = [] for cur_node in cur_layer.nodes: for out_edge in cur_node.out_edges: for suffix in ["", "_pos", "_neg"]: if out_edge.dest + suffix == next_node.name: weight = out_edge.weight edge = Edge(cur_node.name, next_node.name, weight) cur_node.new_out_edges.append(edge) next_node.new_in_edges.append(edge) next_node.in_edges = next_node.new_in_edges del next_node.new_in_edges for node in cur_layer.nodes: node.out_edges = node.new_out_edges del node.new_out_edges if VERBOSE: debug_print("after adjust_layer_after_split_pos_neg()") print(network)
def abstract_network(network: Network, do_preprocess: bool = True, visualize: bool = False, verbose: bool = VERBOSE) -> Network: if VERBOSE: debug_print("original net:") print(network) if do_preprocess: preprocess(network) next_layer_part2union = {} for i in range(len(network.layers) - 1, FIRST_ABSTRACT_LAYER - 1, -1): layer = network.layers[i] next_layer_part2union = layer.abstract(network.name2node_map, next_layer_part2union) # update name2node_map - add new union nodes and remove inner nodes # removal precedes addition for equal names case (e.g output layer) for part in next_layer_part2union.keys(): del network.name2node_map[part] network.generate_name2node_map() # # print (i) # if visualize: # title = "after layer {} test_abstraction".format(i) # visualize_network(network_layers=network.layers, # title=title, # next_layer_part2union=next_layer_part2union, # debug=False) # if verbose: # debug_print("net after abstract {}'th layer:".format(i)) # print(network) finish_abstraction(network, next_layer_part2union, verbose=verbose) return network
def preprocess(network:Network) -> None: """ pre-process networdk in two stages: split to pos/neg, then split to inc/dec :param network: Network before pre-process (nodes without special types) :return: Network after pre-process (nodes with special types pos/neg, inc/dec) """ # split to pos/neg preprocess_split_pos_neg(network) # fill_zero_edges(network) # from core.visualization.visualize_network import visualize_network # split to inc/dec preprocess_split_inc_dec(network) # fill_zero_edges(network) if VERBOSE: debug_print("after preprocess") # visualize_network(network_layers=network.layers, title="after preprocess w/o zero edges") fill_zero_edges(network) # print(network) # self.visualize(title="after preprocess") # visualize_network(network_layers=network.layers, title="after preprocess w/ zero edges", out_image_path="/home/xiaoze/Desktop/images/1.png", debug=True) # generate a copy of the original pre-processed network for later use network.orig_layers = copy.deepcopy(network.layers) network.orig_name2node_map = copy.deepcopy(network.name2node_map) network.weights = network.generate_weights() network.biases = network.generate_biases()
def preprocess_split_inc_dec(network: Network) -> None: """ split net nodes to increasing/decreasing nodes preprocess all layers except input layer (from last to first) """ if VERBOSE: debug_print("preprocess_split_inc_dec()") for i in range(len(network.layers) - 1, FIRST_INC_DEC_LAYER, -1): network.layers[i].split_inc_dec(network.name2node_map) network.generate_name2node_map() adjust_layer_after_split_inc_dec(network, layer_index=FIRST_INC_DEC_LAYER) if VERBOSE: debug_print("after preprocess_split_inc_dec()") print(network)
def preprocess_split_pos_neg(network: Network) -> None: """ split net nodes to nodes with only positive/negative out edges preprocess all hidden layers (from last to first), then adjust input layer """ if VERBOSE: debug_print("preprocess_split_pos_neg()") # orig_input_layer = copy.deepcopy(network.layers[0]) for i in range(len(network.layers) - 2, FIRST_POS_NEG_LAYER, -1): network.layers[i].split_pos_neg(network.name2node_map) # splited_input_layer = self.layers[0] # for node in orig_input_layer.nodes: # node.out_edges = [] # for splitted_node in splited_input_layer: # if splitted_node.name[:-4] == node.name: # suffix=_oos/_neg # edge = Edge(src=node.name, dest=splitted_node.name, weight=1.0) # node.out_edges.append(edge) # splitted_node.in_edges.append(edge) network.generate_name2node_map() # print(self) adjust_layer_after_split_pos_neg(network, layer_index=FIRST_POS_NEG_LAYER)
def one_experiment(nnet_filename, is_tiny, refinement_type, abstraction_type, epsilon, lower_bound, preprocess_orig_net, refinement_sequence_length, abstraction_sequence_length, results_directory, property_id=consts.PROPERTY_ID, verbose=consts.VERBOSE): if verbose: debug_print("one_experiment_cegarabou({})".format(json.dumps([nnet_filename, is_tiny, refinement_type, abstraction_type, epsilon, lower_bound, property_id, preprocess_orig_net]))) if is_tiny: example_nets_dir_path = consts.PATH_TO_MARABOU_ACAS_EXAMPLES else: example_nets_dir_path = consts.PATH_TO_MARABOU_APPLICATIONS_ACAS_EXAMPLES fullname = os.path.join(example_nets_dir_path, nnet_filename) if not os.path.exists(results_directory): os.makedirs(results_directory) results_filename = generate_results_filename(nnet_filename=nnet_filename, is_tiny=is_tiny, refinement_type=refinement_type, abstraction_type=abstraction_type, epsilon=epsilon, lower_bound=lower_bound, property_id=property_id, preprocess_orig_net=preprocess_orig_net, abstraction_sequence_length=abstraction_sequence_length, refinement_sequence_length=refinement_sequence_length) test_property = get_test_property_tiny() if is_tiny else get_test_property_acas(property_id) # for i in range(len(test_property["output"])): # test_property["output"][i][1]["Lower"] = lower_bound net = network_from_nnet_file(fullname) print(f"size={len(net.layers)}") orig_net = copy.deepcopy(net) if args.preprocess_orig_net: preprocess(orig_net) if verbose: print("query using AR") t2 = time.time() if abstraction_type == "complete": print("complete") net = abstract_network(net) else: print("heuristic") net = heuristic_abstract(network=net, test_property=test_property, sequence_length=abstraction_sequence_length) abstraction_time = time.time() - t2 num_of_refine_steps = 0 ar_times = [] ar_sizes = [] refine_sequence_times = [] while True: # CEGAR / CETAR method t4 = time.time() vars1, stats1, query_result = get_query( network=net, test_property=test_property, verbose=consts.VERBOSE ) debug_print(f'query_result={query_result}') t5 = time.time() ar_times.append(t5 - t4) ar_sizes.append(net.get_general_net_data()["num_nodes"]) if verbose: print("query time after A and {} R steps is {}".format(num_of_refine_steps, t5-t4)) debug_print(net.get_general_net_data()) if query_result == "UNSAT": # if always y'<3.99 then also always y<3.99 if verbose: print("UNSAT (finish)") break if query_result == "SAT": if verbose: print("SAT (have to check example on original net)") print(vars1) debug_print(f'vars1={vars1}') st = time.time() orig_net_output = orig_net.evaluate(vars1) print("evaluate: {}".format(time.time() - st)) st = time.time() orig_net.speedy_evaluate(vars1) print("speedy evaluate: {}".format(time.time() - st)) nodes2variables, variables2nodes = orig_net.get_variables() # we got y'>3.99, check if also y'>3.99 for the same input if is_satisfying_assignment(network=orig_net, test_property=test_property, output=orig_net_output, variables2nodes=variables2nodes): if verbose: print("property holds also in orig - SAT (finish)") break # also counter example for orig_net else: t_cur_refine_start = time.time() if verbose: print("property doesn't holds in orig - spurious example") num_of_refine_steps += 1 if verbose: print("refine step #{}".format(num_of_refine_steps)) if refinement_type == "cegar": debug_print("cegar") net = refine(network=net, sequence_length=refinement_sequence_length, example=vars1) else: debug_print("cetar") net = refine(network=net, sequence_length=refinement_sequence_length) t_cur_refine_end = time.time() refine_sequence_times.append(t_cur_refine_end - t_cur_refine_start) t3 = time.time() # time to check property on net with marabou using CEGAR total_ar_time = t3 - t2 if verbose: print("ar query time = {}".format(total_ar_time)) # time to check property on the last network in CEGAR last_net_ar_time = t3 - t4 if verbose: print("last ar net query time = {}".format(last_net_ar_time)) res = [ ("net name", nnet_filename), ("property_id", property_id), ("abstraction_time", abstraction_time), ("query_result", query_result), ("num_of_refine_steps", num_of_refine_steps), ("total_ar_query_time", total_ar_time), ("ar_times", json.dumps(ar_times)), ("ar_sizes", json.dumps(ar_sizes)), ("refine_sequence_times", json.dumps(refine_sequence_times)), ("last_net_data", json.dumps(net.get_general_net_data())), ("last_query_time", last_net_ar_time) ] # generate dataframe from result df = pd.DataFrame.from_dict({x[0]: [x[1]] for x in res}) df.to_json(os.path.join(results_directory, "df_" + results_filename)) # write result to output file with open(os.path.join(results_directory, results_filename), "w") as fw: fw.write("-"*80) fw.write("parameters:") fw.write("-"*80) fw.write("\n") for arg in vars(args): fw.write("{}: {}\n".format(arg, getattr(args, arg))) fw.write("+"*80) fw.write("results:") fw.write("+"*80) fw.write("\n") for (k,v) in res: fw.write("{}: {}\n".format(k,v)) return res
type=int) parser.add_argument("-d", "--results_directory", dest="results_directory", default=consts.results_directory) parser.add_argument("-v", "--verbose", dest="verbose", default=consts.VERBOSE, action="store_true") args = parser.parse_args() return args if __name__ == '__main__': args = parse_args() # run experiment nnet_general_filename = "ACASXU_run2a_1_1_tiny_{}.nnet" if args.is_tiny \ else "ACASXU_run2a_{}_batch_2000.nnet" one_exp_res = one_experiment( nnet_filename=nnet_general_filename.format(args.net_number), property_id=args.property_id, is_tiny=args.is_tiny, refinement_type=args.refinement_type, abstraction_type=args.abstraction_type, epsilon=args.epsilon, lower_bound=args.lower_bound, preprocess_orig_net=args.preprocess_orig_net, abstraction_sequence_length=args.abstraction_sequence_length, refinement_sequence_length=args.refinement_sequence_length, results_directory=args.results_directory) debug_print(one_exp_res)
def one_experiment(nnet_filename, refinement_type, abstraction_type, mechanism, refinement_sequence, abstraction_sequence, results_directory, property_id=consts.PROPERTY_ID, verbose=consts.VERBOSE): test_property = get_test_property_acas(property_id) dynamically_import_marabou(query_type=test_property["type"]) from core.nnet.read_nnet import (network_from_nnet_file, network2rlv) from core.abstraction.naive import abstract_network from core.abstraction.alg2 import heuristic_abstract_alg2 from core.abstraction.random_abstract import heuristic_abstract_random # from core.abstraction.clustering_abstract import \ # heuristic_abstract_clustering from core.utils.marabou_query_utils import reduce_property_to_basic_form, get_query from core.refinement.refine import refine example_nets_dir_path = consts.PATH_TO_MARABOU_APPLICATIONS_ACAS_EXAMPLES fullname = os.path.join(example_nets_dir_path, nnet_filename) if not os.path.exists(results_directory): os.makedirs(results_directory) results_filename = generate_results_filename( nnet_filename=nnet_filename, property_id=property_id, mechanism=mechanism, refinement_type=refinement_type, abstraction_type=abstraction_type, refinement_sequence=refinement_sequence, abstraction_sequence=abstraction_sequence) # for i in range(len(test_property["output"])): # test_property["output"][i][1]["Lower"] = lower_bound net = network_from_nnet_file(fullname) # print(net) print(f"size={len(net.layers)}") net, test_property = reduce_property_to_basic_form( network=net, test_property=test_property) # mechanism is vanilla marabou if mechanism == "marabou": print("query using vanilla Marabou") t0 = time.time() vars1, stats1, query_result = get_query(network=net, test_property=test_property, verbose=consts.VERBOSE) t1 = time.time() # time to check property on net with marabou marabou_time = t1 - t0 if verbose: print(f"query time = {marabou_time}") res = [ ("net_name", nnet_filename), ("property_id", property_id), ("query_result", query_result), ("orig_query_time", marabou_time), ("net_data", json.dumps(net.get_general_net_data())), ] saveResultToFile(results_directory, results_filename, res) return res else: # mechanism is marabou_with_ar orig_net = copy.deepcopy(net) print("query using Marabou with AR") t2 = time.time() if abstraction_type == "complete": net = abstract_network(net) elif abstraction_type == "heuristic_alg2": net = heuristic_abstract_alg2(network=net, test_property=test_property, sequence_length=abstraction_sequence) elif abstraction_type == "heuristic_random": net = heuristic_abstract_random( network=net, test_property=test_property, sequence_length=abstraction_sequence) # elif abstraction_type == "heuristic_clustering": # net = heuristic_abstract_clustering( # network=net, # test_property=test_property, # sequence_length=abstraction_sequence # ) else: raise NotImplementedError("unknown abstraction") abstraction_time = time.time() - t2 num_of_refine_steps = 0 ar_times = [] ar_sizes = [] refine_sequence_times = [] spurious_examples = [] while True: # CEGAR / CETAR method t4 = time.time() print(net.get_general_net_data()) vars1, stats1, query_result = get_query( network=net, test_property=test_property, verbose=consts.VERBOSE) t5 = time.time() ar_times.append(t5 - t4) ar_sizes.append(net.get_general_net_data()["num_nodes"]) # if verbose: print("query time after A and {} R steps is {}".format( num_of_refine_steps, t5 - t4)) debug_print(net.get_general_net_data()) if query_result == "UNSAT": print("+" * 100) # if always y'<3.99 then also always y<3.99 if verbose: print("UNSAT (finish)") break if query_result == "SAT": if verbose: print("SAT (have to check example on original net)") print(vars1) # debug_print(f'vars1={vars1}') # st = time.time() # orig_net_output = orig_net.evaluate(vars1) # print("evaluate: {}".format(time.time() - st)) # st = time.time() orig_net_output = orig_net.speedy_evaluate(vars1) # print(f"orig_net_output={orig_net_output}") # print(f"orig_net.name2node_map={orig_net.name2node_map}") # print("speedy evaluate: {}".format(time.time() - st)) nodes2variables, variables2nodes = orig_net.get_variables() # we got y'>3.99, check if also y'>3.99 for the same input if is_satisfying_assignment(network=orig_net, test_property=test_property, output=orig_net_output, variables2nodes=variables2nodes): if verbose: print("property holds also in orig - SAT (finish)") break # also counter example for orig_net else: spurious_examples.append(vars1) t_cur_refine_start = time.time() if verbose: print( "property doesn't holds in orig - spurious example" ) num_of_refine_steps += 1 if verbose: print("refine step #{}".format(num_of_refine_steps)) # refine until all spurious examples are satisfied # since all spurious examples are satisfied in the original # network, the loop stops until net will be fully refined refinement_sequences_counter = 0 while True: refinement_sequences_counter += 1 # print(f"refinement_sequences_counter={refinement_sequences_counter}") if refinement_type == "cegar": debug_print("cegar") net = refine(network=net, sequence_length=refinement_sequence, example=vars1) else: debug_print("weight_based") net = refine(network=net, sequence_length=refinement_sequence) # after refining, check if the current spurious example is # already not a counter example (i.e. not satisfied in the # refined network). stop if not satisfied, continue if yes net_output = net.speedy_evaluate(vars1) # print(f"net_output={net_output}") # print(f"net.name2node_map={net.name2node_map}") nodes2variables, variables2nodes = net.get_variables() if not is_satisfying_assignment( network=net, test_property=test_property, output=net_output, variables2nodes=variables2nodes): break t_cur_refine_end = time.time() refine_sequence_times.append(t_cur_refine_end - t_cur_refine_start) t3 = time.time() # time to check property on net with marabou using CEGAR total_ar_time = t3 - t2 if verbose: print("ar query time = {}".format(total_ar_time)) # time to check property on the last network in CEGAR last_net_ar_time = t3 - t4 if verbose: print("last ar net query time = {}".format(last_net_ar_time)) res = [("net_name", nnet_filename), ("property_id", property_id), ("abstraction_time", abstraction_time), ("query_result", query_result), ("num_of_refine_steps", num_of_refine_steps), ("total_ar_query_time", total_ar_time), ("ar_times", json.dumps(ar_times)), ("ar_sizes", json.dumps(ar_sizes)), ("refine_sequence_times", json.dumps(refine_sequence_times)), ("last_net_data", json.dumps(net.get_general_net_data())), ("last_query_time", last_net_ar_time)] return res
def one_experiment(nnet_filename, is_tiny, lower_bound, preprocess_orig_net, results_directory, property_id=consts.PROPERTY_ID, is_adversarial_property=False, verbose=consts.VERBOSE): if verbose: debug_print("one_experiment_marabou({})".format( json.dumps([ nnet_filename, is_tiny, lower_bound, preprocess_orig_net, property_id, results_directory ]))) if is_tiny: example_nets_dir_path = consts.PATH_TO_MARABOU_ACAS_EXAMPLES else: example_nets_dir_path = consts.PATH_TO_MARABOU_APPLICATIONS_ACAS_EXAMPLES fullname = os.path.join(example_nets_dir_path, nnet_filename) if not os.path.exists(results_directory): os.makedirs(results_directory) results_filename = generate_results_filename( nnet_filename=nnet_filename, is_tiny=is_tiny, lower_bound=lower_bound, preprocess_orig_net=preprocess_orig_net, property_id=property_id, is_adversarial_property=is_adversarial_property) test_property = get_test_property_tiny( ) if is_tiny else get_test_property_acas(property_id) # for i in range(len(test_property["output"])): # test_property["output"][i][1]["Lower"] = lower_bound net = network_from_nnet_file(fullname) orig_net = copy.deepcopy(net) if args.preprocess_orig_net: preprocess(orig_net) # query original net if verbose: print("query orig_net") t0 = time.time() if verbose: debug_print("orig_net.get_general_net_data(): {}".format( orig_net.get_general_net_data())) vars1, stats1, query_result = get_query( network=orig_net, test_property=test_property, is_adversarial_property=consts.IS_ADVERSARIAL, verbose=consts.VERBOSE) t1 = time.time() # time to check property on net with marabou marabou_time = t1 - t0 if verbose: print("orig_net query time ={}".format(marabou_time)) res = [ ("net name", nnet_filename), ("property_id", property_id), ("query_result", query_result), ("orig_query_time", marabou_time), ("net_data", json.dumps(orig_net.get_general_net_data())), ] # generate dataframe from result df = pd.DataFrame.from_dict({x[0]: [x[1]] for x in res}) df.to_json(os.path.join(results_directory, "df_" + results_filename)) with open(os.path.join(results_directory, results_filename), "w") as fw: fw.write("-" * 80) fw.write("parameters:") fw.write("-" * 80) fw.write("\n") for arg in vars(args): fw.write("{}: {}\n".format(arg, getattr(args, arg))) fw.write("+" * 80) fw.write("results:") fw.write("+" * 80) fw.write("\n") for (k, v) in res: fw.write("{}: {}\n".format(k, v)) return res
default=consts.IS_ADVERSARIAL, type=bool) parser.add_argument("-l", "--lower_bound", dest="lower_bound", default=25000, type=int) parser.add_argument("-d", "--results_directory", dest="results_directory", default=consts.results_directory) parser.add_argument("-v", "--verbose", dest="verbose", default=consts.VERBOSE, action="store_true") args = parser.parse_args() # run experiment nnet_general_filename = "ACASXU_run2a_1_1_tiny_{}.nnet" if args.is_tiny else "ACASXU_run2a_{}_batch_2000.nnet" res_one_exp = one_experiment( nnet_filename=nnet_general_filename.format(args.net_number), is_tiny=args.is_tiny, property_id=args.property_id, preprocess_orig_net=args.preprocess_orig_net, lower_bound=args.lower_bound, results_directory=args.results_directory, is_adversarial_property=args.is_adversarial_property, verbose=args.verbose) debug_print(res_one_exp)
def one_experiment(nnet_filename, property_filename, is_tiny, refinement_type, abstraction_type, epsilon, lower_bound, orig_net_query, preprocess_orig_net, refinement_sequence_length, abstraction_sequence_length, results_directory): debug_print("one_experiment({})".format(json.dumps([nnet_filename, is_tiny, refinement_type, abstraction_type, epsilon, lower_bound, orig_net_query, preprocess_orig_net]))) if is_tiny: example_nets_dir_path = consts.PATH_TO_MARABOU_ACAS_EXAMPLES else: example_nets_dir_path = consts.PATH_TO_MARABOU_APPLICATIONS_ACAS_EXAMPLES fullname = os.path.join(example_nets_dir_path, nnet_filename) if not os.path.exists(results_directory): os.makedirs(results_directory) results_filename = generate_results_filename(nnet_filename=nnet_filename, property_filename=property_filename, is_tiny=is_tiny, refinement_type=refinement_type, abstraction_type=abstraction_type, epsilon=epsilon, lower_bound=lower_bound, orig_net_query=orig_net_query, preprocess_orig_net=preprocess_orig_net, abstraction_sequence_length=abstraction_sequence_length, refinement_sequence_length=refinement_sequence_length) test_property = read_test_property(property_filename) for i in range(len(test_property["output"])): test_property["output"][i][1]["Lower"] = lower_bound net = network_from_nnet_file(fullname) orig_net = copy.deepcopy(net) if args.preprocess_orig_net: orig_net.preprocess() # query original net if args.orig_net_query: print("query orig_net") t0 = time.time() debug_print("orig_net.get_general_net_data(): {}".format(orig_net.get_general_net_data())) vars1, stats1, query_result = orig_net.get_query(test_property) t1 = time.time() # time to check property on net with marabou marabou_time = t1 - t0 print("orig_net query time ={}".format(marabou_time)) else: marabou_time = None print("query using AR") t2 = time.time() if abstraction_type == "complete": net = net.abstract() else: net = net.heuristic_abstract(test_property=test_property, sequence_length=abstraction_sequence_length) abstraction_time = time.time() - t2 num_of_refine_steps = 0 ar_times = [] ar_sizes = [] refine_sequence_times = [] while True: # CEGAR / CETAR method t4 = time.time() vars1, stats1, query_result = net.get_query(test_property) t5 = time.time() ar_times.append(t5 - t4) ar_sizes.append(net.get_general_net_data()["num_nodes"]) print("query time after A and {} R steps is {}".format(num_of_refine_steps, t5-t4)) debug_print(net.get_general_net_data()) if query_result == "UNSAT": # if always y'<3.99 then also always y<3.99 print("UNSAT (finish)") break if query_result == "SAT": print("SAT (have to check example on original net)") print(vars1) orig_net_output = orig_net.evaluate(vars1) nodes2variables, variables2nodes = orig_net.get_variables() # we got y'>3.99, check if also y'>3.99 for the same input if orig_net.does_property_holds(test_property, orig_net_output, variables2nodes): print("property holds also in orig - SAT (finish)") break # also counter example for orig_net else: t_cur_refine_start = time.time() print("property doesn't holds in orig - spurious example") num_of_refine_steps += 1 print("refine step #{}".format(num_of_refine_steps)) if refinement_type == "cegar": net = net.refine(sequence_length=refinement_sequence_length, example=vars1) else: net = net.refine(sequence_length=refinement_sequence_length) t_cur_refine_end = time.time() refine_sequence_times.append(t_cur_refine_end - t_cur_refine_start) t3 = time.time() # time to check property on net with marabou using CEGAR total_ar_time = t3 - t2 print("ar query time = {}".format(total_ar_time)) # time to check property on the last network in CEGAR last_net_ar_time = t3 - t4 print("last ar net query time = {}".format(last_net_ar_time)) res = [ ("net name", nnet_filename), ("abstraction_time", abstraction_time), ("query_result", query_result), ("orig_query_time", marabou_time), ("num_of_refine_steps", num_of_refine_steps), ("total_ar_query_time", total_ar_time), ("ar_times", json.dumps(ar_times)), ("ar_sizes", json.dumps(ar_sizes)), ("refine_sequence_times", json.dumps(refine_sequence_times)), ("last_net_data", json.dumps(net.get_general_net_data())), ("last_query_time", last_net_ar_time) ] with open(os.path.join(results_directory, results_filename), "w") as fw: fw.write("-"*80) fw.write("parameters:") fw.write("-"*80) fw.write("\n") for arg in vars(args): fw.write("{}: {}\n".format(arg, getattr(args, arg))) fw.write("+"*80) fw.write("results:") fw.write("+"*80) fw.write("\n") for (k,v) in res: fw.write("{}: {}\n".format(k,v)) return res
default=refinement_type_default, choices=["cegar", "cetar"]) parser.add_argument("-p", "--preprocess_orig_net", dest="preprocess_orig_net", default=consts.COMPARE_TO_PREPROCESSED_NET, action="store_true") parser.add_argument("-o", "--orig_net_query", dest="orig_net_query", default=False, action="store_true") parser.add_argument("-e", "--epsilon", dest="epsilon", default=consts.EPSILON, type=float) parser.add_argument("-l", "--lower_bound", dest="lower_bound", default=25000, type=int) parser.add_argument("-as", "--abstraction_sequence_length", dest="abstraction_sequence_length", default=100, type=int) parser.add_argument("-rs", "--refinement_sequence_length", dest="refinement_sequence_length", default=100, type=int) parser.add_argument("-d", "--results_directory", dest="results_directory", default=consts.results_directory) parser.add_argument("-v", "--verbose", dest="verbose", default=consts.VERBOSE, action="store_true") args = parser.parse_args() # run experiment res = one_experiment(nnet_filename=args.nnet_filename, property_filename=args.property_filename, is_tiny=args.is_tiny, refinement_type=args.refinement_type, abstraction_type=args.abstraction_type, epsilon=args.epsilon, lower_bound=args.lower_bound, orig_net_query=args.orig_net_query, preprocess_orig_net=args.preprocess_orig_net, abstraction_sequence_length=args.abstraction_sequence_length, refinement_sequence_length=args.refinement_sequence_length, results_directory=args.results_directory) debug_print(res)
def split_back(network: Network, part: AnyStr) -> None: """ implement the refinement step. split back part from the union node it was grouped into into a separated node :param network: Network :param part: Str of the name of the original node that is part of the union """ # assume that layer_index is in [2, ..., L-1] (L = num of layers) try: layer_index = int(part.split("_")[1]) except IndexError: debug_print("IndexError in core.test_refinement.step.split_back()") import IPython IPython.embed() layer = network.layers[layer_index] next_layer = network.layers[layer_index + 1] prev_layer = network.layers[layer_index - 1] part2node_map = network.get_part2node_map() union_node = network.name2node_map[part2node_map[part]] parts = union_node.name.split("+") other_parts = [p for p in parts if p != part] if not other_parts: return part_node = ARNode(name=part, ar_type=union_node.ar_type, activation_func=union_node.activation_func, in_edges=[], out_edges=[], bias=network.orig_name2node_map[part].bias) bias = sum([ network.orig_name2node_map[other_part].bias for other_part in other_parts ]) other_parts_node = ARNode(name="+".join(other_parts), ar_type=union_node.ar_type, activation_func=union_node.activation_func, in_edges=[], out_edges=[], bias=bias) splitting_nodes = [part_node, other_parts_node] for splitting_node in splitting_nodes: # print("splitting_node.name={}".format(splitting_node.name)) for next_layer_node in next_layer.nodes: group_a = splitting_node.name.split("+") group_b = next_layer_node.name.split("+") # print("call 1 - group_a") # print(group_a) # print("call 1 - group_b") # print(group_b) out_edge_weight = calculate_weight_of_edge_between_two_part_groups( network=network, group_a=group_a, group_b=group_b) if out_edge_weight is not None: out_edge = Edge(splitting_node.name, next_layer_node.name, out_edge_weight) splitting_node.out_edges.append(out_edge) next_layer_node.in_edges.append(out_edge) # fill_zero_edges(network) for prev_layer_node in prev_layer.nodes: group_a = prev_layer_node.name.split("+") group_b = splitting_node.name.split("+") # print("call 2 - group_a") # print(group_a) # print("call 2 - group_b") # print(group_b) in_edge_weight = calculate_weight_of_edge_between_two_part_groups( network=network, group_a=group_a, group_b=group_b) if in_edge_weight is not None: in_edge = Edge(prev_layer_node.name, splitting_node.name, in_edge_weight) splitting_node.in_edges.append(in_edge) prev_layer_node.out_edges.append(in_edge) # fill_zero_edges(network) layer.nodes.append(splitting_node) fill_zero_edges(network) network.remove_node(union_node, layer_index) network.generate_name2node_map()
def planet_with_ar(json_content): print(json_content) parameter = json.loads(json_content) refinement_type = parameter['refinement_type'] abstraction_type = parameter['abstract_type'] refinement_sequence = int(parameter['refinement_sequence']) abstraction_sequence = int(parameter['abstraction_sequence']) test_property = generate_test_property(parameter) net = network_from_nnet_file(parameter['filepath']) net, test_property = reduce_property_to_basic_form( network=net, test_property=test_property) orig_net = copy.deepcopy(net) # Abstract t2 = time.time() if abstraction_type == "complete": net = abstract_network(net) elif abstraction_type == "heuristic_alg2": net = heuristic_abstract_alg2(network=net, test_property=test_property, sequence_length=abstraction_sequence) elif abstraction_type == "heuristic_random": net = heuristic_abstract_random(network=net, test_property=test_property, sequence_length=abstraction_sequence) # elif abstraction_type == "heuristic_clustering": # net = heuristic_abstract_clustering( # network=net, # test_property=test_property, # sequence_length=abstraction_sequence # ) else: raise NotImplementedError("unknown abstraction") abstraction_time = time.time() - t2 network2rlv(net, test_property, "network.rlv") shell_cmd = "{} network.rlv".format(parameter['planet']) cmd = shlex.split(shell_cmd) print("here") num_of_refine_steps = 0 ar_times = [] ar_sizes = [] refine_sequence_times = [] spurious_examples = [] while True: t4 = time.time() print(net.get_general_net_data()) p = subprocess.Popen(cmd, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, err = p.communicate() res = out.decode(encoding='utf-8').strip().split('\n') query_result = res[0] if 'UNSAT' == query_result: break if 'SAT' == query_result: # counterexample vars1 = {} for i in range(5): content = res[3 + i].split('/')[-1].strip() vars1[i] = float(content) orig_net_output = orig_net.speedy_evaluate(vars1) nodes2variables, variables2nodes = orig_net.get_variables() # we got y'>3.99, check if also y'>3.99 for the same input if is_satisfying_assignment(network=orig_net, test_property=test_property, output=orig_net_output, variables2nodes=variables2nodes): print("property holds also in orig - SAT (finish)") break # also counter example for orig_net else: print("need to refine") spurious_examples.append(vars1) num_of_refine_steps += 1 t_cur_refine_start = time.time() refinement_sequences_counter = 0 while True: refinement_sequences_counter += 1 if refinement_type == "cegar": debug_print("cegar") net = refine(network=net, sequence_length=refinement_sequence, example=vars1) else: debug_print("weight_based") net = refine(network=net, sequence_length=refinement_sequence) # update network file network2rlv(net, test_property, "network.rlv") net_output = net.speedy_evaluate(vars1) nodes2variables, variables2nodes = net.get_variables() if not is_satisfying_assignment( network=orig_net, test_property=test_property, output=orig_net_output, variables2nodes=variables2nodes): break t_cur_refine_end = time.time() refine_sequence_times.append(t_cur_refine_end - t_cur_refine_start) t3 = time.time() total_ar_time = t3 - t2 last_net_ar_time = t3 - t4 res = { "query_result": query_result, "num_of_refine_steps": str(num_of_refine_steps), "ar_times": str(ar_times), "ar_sizes": str(ar_sizes), "refine_sequence_times": str(refine_sequence_times), } return json.dumps(res)
def verify_with_ar(abstract_net, orig_net, test_property, refinement_type, abstraction_type, refinement_sequence, abstraction_sequence, verbose=consts.VERBOSE): try: # mechanism is marabou_with_ar dynamically_import_marabou(query_type=test_property["type"]) net = copy.deepcopy(abstract_net) orig_net = copy.deepcopy(orig_net) test_property = copy.deepcopy(test_property) num_of_refine_steps = 0 ar_times = [] ar_sizes = [] refine_sequence_times = [] spurious_examples = [] start = time.time() while True: # CEGAR / CETAR method t4 = time.time() vars1, stats1, query_result = get_query( network=net, test_property=test_property, verbose=consts.VERBOSE) t5 = time.time() ar_times.append(t5 - t4) ar_sizes.append(net.get_general_net_data()["num_nodes"]) # if verbose: print("query time after A and {} R steps is {}".format( num_of_refine_steps, t5 - t4)) if query_result == "UNSAT": # if always y'<3.99 then also always y<3.99 if verbose: print("UNSAT (finish)") break if query_result == "SAT": if verbose: print("SAT (have to check example on original net)") orig_net_output = orig_net.speedy_evaluate(vars1) nodes2variables, variables2nodes = orig_net.get_variables() # we got y'>3.99, check if also y'>3.99 for the same input if is_satisfying_assignment(network=orig_net, test_property=test_property, output=orig_net_output, variables2nodes=variables2nodes): if verbose: print("property holds also in orig - SAT (finish)") break # also counter example for orig_net else: spurious_examples.append(vars1) t_cur_refine_start = time.time() if verbose: print( "property doesn't holds in orig - spurious example" ) num_of_refine_steps += 1 if verbose: print("refine step #{}".format(num_of_refine_steps)) # refine until all spurious examples are satisfied # since all spurious examples are satisfied in the original # network, the loop stops until net will be fully refined refinement_sequences_counter = 0 while True: refinement_sequences_counter += 1 # print(f"refinement_sequences_counter={refinement_sequences_counter}") if refinement_type == "cegar": debug_print("cegar") net = refine(network=net, sequence_length=refinement_sequence, example=vars1) else: debug_print("weight_based") net = refine(network=net, sequence_length=refinement_sequence) # after refining, check if the current spurious example is # already not a counter example (i.e. not satisfied in the # refined network). stop if not satisfied, continue if yes net_output = net.speedy_evaluate(vars1) # print(f"net_output={net_output}") # print(f"net.name2node_map={net.name2node_map}") nodes2variables, variables2nodes = net.get_variables() if not is_satisfying_assignment( network=net, test_property=test_property, output=net_output, variables2nodes=variables2nodes): break t_cur_refine_end = time.time() refine_sequence_times.append(t_cur_refine_end - t_cur_refine_start) t3 = time.time() consume = t3 - start # time to check property on the last network in CEGAR last_net_ar_time = t3 - t4 if verbose: print("last ar net query time = {}".format(last_net_ar_time)) res = { "query_result": query_result, "num_of_refine_steps": str(num_of_refine_steps), "ar_times": str(ar_times), "ar_sizes": str(ar_sizes), "refine_sequence_times": str(refine_sequence_times), "refinement_consume": str(consume) } return json.dumps(res) except Exception as e: print("exception occur", e)