def verify(eran_model_path, input, epsilon): model, is_conv, means, stds = read_net(eran_model_path, 14, False) eran = ERAN(model) specLB = np.clip(input - epsilon, 0, 1) specUB = np.clip(input + epsilon, 0, 1) for i in range(0, 2): specLB[i] = input[i] specUB[i] = input[i] for i in range(2, 4): specLB[i] = np.clip(input[i] - epsilon, -1, 1) specUB[i] = np.clip(input[i] + epsilon, -1, 1) label, nn, _, _ = eran.analyze_box(specLB, specUB, 'deepzono', 1, 1) print("lable = {}".format(label)) return label >= 0
def verify(dnn: OperationGraph, phi: Expression, domain="deeppoly", timeout_lp=1.0, timeout_milp=1.0, use_area_heuristic=True, **kwargs: Dict[str, Any]): logger = logging.getLogger(__name__) dnn = dnn.simplify() phi.networks[0].concretize(dnn) result = UNSAT property_extractor = HalfspacePolytopePropertyExtractor( HyperRectangle, HalfspacePolytope) for prop in property_extractor.extract_from(~phi): if prop.input_constraint.num_variables > 1: raise ERANTranslatorError( "Unsupported network: More than 1 input variable") with tf.Session(graph=tf.Graph()) as tf_session: layers = as_layers( prop.suffixed_op_graph(), extra_layer_types=ERAN_LAYER_TYPES, translator_error=ERANTranslatorError, ) input_interval = prop.input_constraint spec_lb = input_interval.lower_bounds[0] spec_ub = input_interval.upper_bounds[0] if len(spec_lb.shape) == 4: spec_lb = spec_lb.transpose((0, 2, 3, 1)) spec_ub = spec_ub.transpose((0, 2, 3, 1)) tf_graph = as_tf(layers, translator_error=ERANTranslatorError) eran_model = ERAN(tf_graph, session=tf_session) _, nn, nlb, nub = eran_model.analyze_box(spec_lb.flatten().copy(), spec_ub.flatten().copy(), domain, timeout_lp, timeout_milp, use_area_heuristic, **kwargs) output_lower_bound = np.asarray(nlb[-1]) output_upper_bound = np.asarray(nub[-1]) logger.debug("output lower bound: %s", output_lower_bound) logger.debug("output upper bound: %s", output_upper_bound) result |= check(output_lower_bound, output_upper_bound) if result == SAT or result == UNKNOWN: return result return result
def runMNIST(): nnets = [ nnetPrefix + "/mnist_relu_3_100_16bit.pyt", ] perturb = 3 epsilon = 1.0 numInputs = 784 csvfile = open('../data/mnist_test.csv', 'r') images = csv.reader(csvfile, delimiter=',') tests = [] for i, image in enumerate(images): if i == 10: break target = int(image[0]) image = np.float64(image[1:]) specLB = np.copy(image) specUB = np.copy(image) specLB = np.clip(specLB - perturb, 0, 255) specUB = np.clip(specUB + perturb, 0, 255) specLB = specLB / np.float64(255) specUB = specUB / np.float64(255) tests.append((target, specLB, specUB)) for nnet in nnets: print("Network: " + nnet.split("/")[-1], file=sys.stderr) model, is_conv, means, stds = read_net(nnet, numInputs, False) eran = ERAN(model) for target, lb, ub in tests: res = getEps2(eran, lb, ub, target, 10) print("Result: " + str(res), file=sys.stderr) if abs(res[1]) < epsilon and abs(res[2]) < epsilon: print("Verified", file=sys.stderr) else: print("Failed", file=sys.stderr) exit(0)
def runHAR(): nnet = nnetPrefix + "/HAR_16bit.pyt" perturb = 0.1 epsilon = 0.25 numInputs = 561 csvfile = open('../data/har_test.csv', 'r') testInputs = csv.reader(csvfile, delimiter=',') tests = [] for i, testInput in enumerate(testInputs): if i == 10: break target = int(testInput[0]) testInput = np.float64(testInput[1:]) specLB = np.copy(testInput) specUB = np.copy(testInput) specLB = np.clip(specLB - perturb, -1, 1) specUB = np.clip(specUB + perturb, -1, 1) tests.append((target, specLB, specUB)) model, is_conv, means, stds = read_net(nnet, numInputs, False) eran = ERAN(model) for target, lb, ub in tests: res = getEps2(eran, lb, ub, target, 6) print("Result: " + str(res), file=sys.stderr) if abs(res[1]) < epsilon and abs(res[2]) < epsilon: print("Verified", file=sys.stderr) else: print("Failed", file=sys.stderr) exit(0)
def runACAS(): epsilon = 0.05 numInputs = 5 nnet = "ACASXU_run2a_1_1_batch_2000_16bit.pyt" nnetPath = nnetPrefix + nnet model, is_conv, means, stds = read_net(nnetPath, numInputs, False) eran = ERAN(model) target = 0 specUB = prop16UB specLB = prop16LB print("Property: " + str(16)) print("Network: " + nnet) res = getEps2(eran, specLB, specUB, target, 5) print("Result: " + str(res)) if abs(res[1]) < epsilon and abs(res[2]) < epsilon: print("Verified") else: print("Failed") specUB = prop26UB specLB = prop26LB print("Property: " + str(26)) print("Network: " + nnet) res = getEps2(eran, specLB, specUB, target, 5) print("Result: " + str(res)) if abs(res[1]) < epsilon and abs(res[2]) < epsilon: print("Verified") else: print("Failed")
def getERAN(netname, num_pixels): is_trained_with_pytorch = True domain = 'refinepoly' model, is_conv, means, stds = read_tensorflow_net( netname, num_pixels, is_trained_with_pytorch, (domain == 'gpupoly' or domain == 'refinegpupoly')) eran = ERAN(model, is_onnx=False) return eran
def _start_acasxu_recursive(kwargs): """ Utility method to start _acasxu_recursive through multiprocessing """ network_file = kwargs['network_file'] model, _ = read_onnx_net(network_file) eran = ERAN(model, is_onnx=True) global _failed_already return _acasxu_recursive( kwargs['specLB'], kwargs['specUB'], model, eran, kwargs['constraints'], _failed_already, kwargs['max_depth'], 0, kwargs['permitted_depth_extensions'], 0, kwargs['domain'], kwargs['timeout_lp'], kwargs['timeout_milp'], kwargs['use_default_heuristic'], kwargs['complete'])
def runACAS(): epsilon = 0.05 numInputs = 5 erans = {} for nnet in ACASNnets: nnetPath = nnetPrefix + nnet model, is_conv, means, stds = read_net(nnetPath, numInputs, False) erans[nnet] = ERAN(model) for prop, test in ACASProps.items(): target, specUB, specLB = test print("Property: " + str(prop)) for nnet in ACASNnets: if re.search(ACASExps[prop], nnet) and not re.search(ACASSkips[prop], nnet): print("Network: " + nnet) res = getEps2(erans[nnet], specLB, specUB, target, 5) print("Result: " + str(res)) if abs(res[1]) < epsilon and abs(res[2]) < epsilon: print("Verified") else: print("Failed")
def __analyze_segmentation_eran(eran: ERAN, true_labels, valid_classes, num_total_classes, specLB, specUB, domain, timeout_lp, timeout_milp, use_default_heuristic, lexpr_weights=None, lexpr_cst=None, lexpr_dim=None, uexpr_weights=None, uexpr_cst=None, uexpr_dim=None, expr_size=0, testing=False, prop=-1, spatial_constraints=None): assert domain == "deeppoly", "domain isn't valid, must be 'deeppoly'" specLB = np.reshape(specLB, (-1, )) specUB = np.reshape(specUB, (-1, )) eran.nn = layers() eran.nn.specLB = specLB eran.nn.specUB = specUB execute_list, output_info = eran.optimizer.get_deeppoly( eran.nn, specLB, specUB, lexpr_weights, lexpr_cst, lexpr_dim, uexpr_weights, uexpr_cst, uexpr_dim, expr_size, spatial_constraints) analyzer = Analyzer(execute_list, eran.nn, domain, timeout_lp, timeout_milp, None, use_default_heuristic, -1, prop, testing) dominant_classes, nlb, nub = EranVerifier.__analyze_segmentation_analyzer( analyzer, true_labels, valid_classes, num_total_classes) return dominant_classes, nlb, nub
def runMNIST3Pixel(): nnets = [ nnetPrefix + "/mnist_relu_3_100_16bit.pyt", nnetPrefix + "/mnist_relu_2_512_16bit.pyt", nnetPrefix + "/mnist_relu_4_1024_16bit.pyt" ] perturb = 3 epsilon = 1.0 numInputs = 784 numPixels = 3 csvfile = open('../data/mnist_test.csv', 'r') images = csv.reader(csvfile, delimiter=',') tests = [] for i, image in enumerate(images): target = int(image[0]) image = np.float64(image[1:]) specLB = np.copy(image) specUB = np.copy(image) for j in range(numPixels): specLB[random_pixels[i][j]] = 0.0 specUB[random_pixels[i][j]] = 255.0 specLB = specLB / np.float64(255) specUB = specUB / np.float64(255) tests.append((target, specLB, specUB)) for nnet in nnets: print("Network: " + nnet.split("/")[-1], file=sys.stderr) model, is_conv, means, stds = read_net(nnet, numInputs, False) eran = ERAN(model) for target, lb, ub in tests: res = getEps2(eran, lb, ub, target, 10) print("Result: " + str(res), file=sys.stderr) if abs(res[1]) < epsilon and abs(res[2]) < epsilon: print("Verified", file=sys.stderr) else: print("Failed", file=sys.stderr)
sess = tf.Session() if is_saved_tf_model: saver = tf.train.import_meta_graph(netname) saver.restore(sess, tf.train.latest_checkpoint(netfolder + '/')) else: with tf.gfile.GFile(netname, "rb") as f: graph_def = tf.GraphDef() graph_def.ParseFromString(f.read()) sess.graph.as_default() tf.graph_util.import_graph_def(graph_def, name='') ops = sess.graph.get_operations() last_layer_index = -1 while ops[last_layer_index].type in non_layer_operation_types: last_layer_index -= 1 eran = ERAN( sess.graph.get_tensor_by_name(ops[last_layer_index].name + ':0'), sess) else: if (dataset == 'mnist'): num_pixels = 784 elif (dataset == 'cifar10'): num_pixels = 3072 elif (dataset == 'acasxu'): num_pixels = 5 elif (zonotope_bool == True): num_pixels = len(zonotope) if is_onnx: is_trained_with_pytorch = True model, is_conv = read_onnx_net(netname) # this is to have different defaults for mnist and cifar10 if dataset == 'cifar10':
def verify_acasxu_style( network_file: str, means: np.ndarray, stds: np.ndarray, input_box: List[Tuple[np.ndarray, np.ndarray]], output_constraints: List[List[Tuple[int, int, float]]], timeout_lp=1, timeout_milp=1, max_depth=10, permitted_depth_extensions=3, use_default_heuristic=True, complete=True, progress_bar=True ) -> Optional[List[Tuple[np.ndarray, Tuple[int, int, float], Optional[float]]]]: """ Verifies an fully-connected (non-convolutional) neural network. Returns None if an no counterexamples could be found and the network could not be verified. Returns an empty list if the network could be verified. Returns a list of inputs for which the network violates the constraints if the network could not be verified. """ domain = "deeppoly" model, is_conv = read_onnx_net(network_file) eran = ERAN(model, is_onnx=True) # total will be updated later progress_bar = tqdm(total=1, disable=not progress_bar) specLB = [interval[0] for interval in input_box] specUB = [interval[1] for interval in input_box] _normalize(specLB, means, stds) _normalize(specUB, means, stds) counterexample_list = [] # adex_holds stores whether x_adex (below) is actually a counterexample # if adex_holds is True, then x_adex is a spurious counterexample adex_holds = True verified_flag, nn, nlb, nub, _, x_adex = eran.analyze_box( specLB, specUB, _init_domain(domain), timeout_lp, timeout_milp, use_default_heuristic, output_constraints) if not verified_flag and x_adex is not None: adex_holds, _, _, _, _, _ = eran.analyze_box(x_adex, x_adex, "deeppoly", timeout_lp, timeout_milp, use_default_heuristic, output_constraints) if not adex_holds: verified_flag = False # we need to undo the input normalisation, that was applied to the counterexamples counterexample_list.append( (np.array(x_adex) * stds + means, output_constraints)) if not verified_flag and adex_holds: # expensive min/max gradient calculation nn.set_last_weights(output_constraints) grads_lower, grads_upper = nn.back_propagate_gradient(nlb, nub) assert len(grads_lower) == len(specLB), 'back_propagate_gradient did not yield gradients for inputs. ' \ '(verify_acasxu_style only supports non-convolutional networks).' smears = [ max(-grad_l, grad_u) * (u - l) for grad_l, grad_u, l, u in zip( grads_lower, grads_upper, specLB, specUB) ] smears = [1e-8 if smear == 0 else smear for smear in smears] split_multiple = 20 / np.sum(smears) num_splits = [int(np.ceil(smear * split_multiple)) for smear in smears] step_size = [] for i in range(len(specLB)): # for each input dimension if num_splits[i] == 0: num_splits[i] = 1 step_size.append((specUB[i] - specLB[i]) / num_splits[i]) start_val = np.copy(specLB) end_val = np.copy(specUB) # _, nn, _, _, _, _ = eran.analyze_box( # specLB, specUB, domain, # timeout_lp, timeout_milp, use_default_heuristic, output_constraints # ) # generate all combinations of splits of the input dimensions multi_bounds = [(specLB.copy(), specUB.copy())] for d in range(len(specLB)): # for each input dimension # for each split from the previous dimensions new_multi_bounds = [] for specLB_, specUB_ in multi_bounds: for i in range(num_splits[d]): specLB_ = specLB_.copy() specUB_ = specUB_.copy() specLB_[d] = start_val[d] + i * step_size[d] specUB_[d] = np.fmin(end_val[d], start_val[d] + (i + 1) * step_size[d]) new_multi_bounds.append((specLB_, specUB_)) multi_bounds = new_multi_bounds # print(f"len(multi_bounds)={len(multi_bounds)}\n" # f"means={means}\n" # f"stds={stds}\n" # f"grads_lower={grads_lower}\n" # f"grads_upper={grads_upper}\n" # f"smeans={smears}\n" # f"num_splits={num_splits}\n" # f"step_size={step_size}\n" # f"start_val={start_val}\n" # f"end_val={end_val}") progress_bar.reset(total=len(multi_bounds) + 1) progress_bar.update() # for the first analyze_box run failed_already = Value('i', 1) pool = None try: # sequential version # res = itertools.starmap( # lambda lb, ub: _acasxu_recursive(lb, ub, model, eran, output_constraints, failed_already, # 25, 0, domain, timeout_lp, timeout_milp, use_default_heuristic, # complete), # multi_bounds # ) arguments = [{ 'specLB': lb, 'specUB': ub, 'network_file': network_file, 'constraints': output_constraints, 'max_depth': max_depth, 'permitted_depth_extensions': permitted_depth_extensions, 'domain': domain, 'timeout_lp': timeout_lp, 'timeout_milp': timeout_milp, 'use_default_heuristic': use_default_heuristic, 'complete': complete } for lb, ub in multi_bounds] # using only half of the CPUs sped up the computation on the computers it was tested on # and also kept CPU utilisation high for all CPUs. pool = Pool(processes=os.cpu_count() // 2, initializer=_init, initargs=(failed_already, )) res = pool.imap_unordered(_start_acasxu_recursive, arguments) counterexample_list = [] verified_flag = True for verified, counterexamples in res: if not verified: verified_flag = False if counterexamples is not None: # convert counterexamples to numpy counterexamples = [ (np.array(cx), constraint, value) for cx, constraint, value in counterexamples ] # we need to undo the input normalisation, that was applied to the counterexamples counterexamples = [ (cx * stds + means, constraint, value) for cx, constraint, value in counterexamples ] counterexample_list.extend(counterexamples) else: warning( f"Property not verified for without counterexample." ) progress_bar.update() except Exception as ex: warning(f"Property not verified because of an exception: {ex}.") raise ex finally: progress_bar.close() if pool is not None: # make sure the Pool is properly closed pool.terminate() pool.join() else: # property has been verified in first analyze_box run. progress_bar.update() progress_bar.close() if not verified_flag and len(counterexample_list) > 0: info( f"Property not verified with counterexamples: {counterexample_list}." ) return counterexample_list elif not verified_flag: info(f"Property not verified without counterexamples.") # raise RuntimeError("Property disproven, but no counterexample found.") return None else: info(f"Property verified.") return []
def init_model(): global netname, epsilon, domain, dataset, means, stds, is_conv netname = config.netname filename, file_extension = os.path.splitext(netname) assert config.netname, 'a network has to be provided for analysis.' is_trained_with_pytorch = file_extension == ".pyt" is_saved_tf_model = file_extension == ".meta" is_pb_file = file_extension == ".pb" is_tensorflow = file_extension == ".tf" is_onnx = file_extension == ".onnx" assert is_trained_with_pytorch or is_saved_tf_model or is_pb_file or is_tensorflow or is_onnx, "file extension not supported" epsilon = config.epsilon domain = config.domain assert domain in [ 'deepzono', 'refinezono', 'deeppoly', 'refinepoly', 'gpupoly', 'refinegpupoly' ], "domain name can be either deepzono, refinezono, deeppoly, refinepoly, gpupoly, refinegpupoly" dataset = config.dataset #assert dataset in ['mnist', 'cifar10', 'acasxu', 'fashion'], "only mnist, cifar10, acasxu, and fashion datasets are supported" print("netname ", netname, " epsilon ", epsilon, " domain ", domain, " dataset ", dataset, "args complete ", config.complete, " complete ", complete, " timeout_lp ", config.timeout_lp) if (dataset == 'mnist'): num_pixels = 784 elif (dataset == 'cifar10'): num_pixels = 3072 elif (dataset == 'acasxu'): num_pixels = 5 else: num_pixels = dataset if is_onnx: model, is_conv = read_onnx_net(netname) else: model, is_conv, means, stds = read_tensorflow_net( netname, num_pixels, is_trained_with_pytorch, (domain == 'gpupoly' or domain == 'refinegpupoly')) if domain == 'gpupoly' or domain == 'refinegpupoly': if is_onnx: translator = ONNXTranslator(model, True) else: translator = TFTranslator(model) operations, resources = translator.translate() optimizer = Optimizer(operations, resources) nn = layers() network, relu_layers, num_gpu_layers = optimizer.get_gpupoly(nn) else: eran = ERAN(model, is_onnx=is_onnx) if not is_trained_with_pytorch: if dataset == 'mnist' and not config.geometric: means = [0] stds = [1] elif dataset == 'acasxu': means = [1.9791091e+04, 0.0, 0.0, 650.0, 600.0] stds = [60261.0, 6.28318530718, 6.28318530718, 1100.0, 1200.0] else: means = [0] stds = [1] is_trained_with_pytorch = is_trained_with_pytorch or is_onnx if config.mean is not None: means = config.mean stds = config.std return eran
def main(): parser = argparse.ArgumentParser(description='Analyze NN.') parser.add_argument('--net', type=str, help='Neural network to analyze') parser.add_argument('--dataset', type=str, default='mnist', help='Dataset') parser.add_argument('--data_dir', type=str, help='Directory which contains data') parser.add_argument('--data_root', type=str, help='Directory which contains data') parser.add_argument('--num_params', type=int, default=0, help='Number of transformation parameters') parser.add_argument('--num_tests', type=int, default=None, help='Number of images to test') parser.add_argument('--from_test', type=int, default=0, help='Number of images to test') parser.add_argument('--test_idx', type=int, default=None, help='Index to test') parser.add_argument('--debug', action='store_true', help='Whether to display debug info') parser.add_argument('--attack', action='store_true', help='Whether to attack') parser.add_argument('--timeout_lp', type=float, default=1, help='timeout for the LP solver') parser.add_argument('--timeout_milp', type=float, default=1, help='timeout for the MILP solver') parser.add_argument( '--use_area_heuristic', type=str2bool, default=True, help='whether to use area heuristic for the DeepPoly ReLU approximation' ) args = parser.parse_args() #LRJ add deepsymbol_input_folder = args.data_dir + '/Batch_Input_DeepG' #print(deepsymbol_input_folder) if not os.path.exists(deepsymbol_input_folder): os.makedirs(deepsymbol_input_folder) else: for root, dirs, files in os.walk(deepsymbol_input_folder): for name in files: if name.endswith('.in'): os.remove(os.path.join(root, name)) #print("delete "+os.path.join(root,name)) global n_rows, n_cols, n_channels if args.dataset == 'cifar10': n_rows, n_cols, n_channels = 32, 32, 3 else: n_rows, n_cols, n_channels = 28, 28, 1 filename, file_extension = os.path.splitext(args.net) is_trained_with_pytorch = False is_saved_tf_model = False if (file_extension == ".net" or file_extension == ".pyt"): is_trained_with_pytorch = True elif (file_extension == ".meta"): is_saved_tf_model = True elif (file_extension != ".tf"): print("file extension not supported") exit(1) is_conv = False if (is_saved_tf_model): netfolder = os.path.dirname(args.net) tf.logging.set_verbosity(tf.logging.ERROR) sess = tf.Session() saver = tf.train.import_meta_graph(args.net) saver.restore(sess, tf.train.latest_checkpoint(netfolder + '/')) eran = ERAN(sess.graph.get_tensor_by_name('logits:0'), sess) else: if args.dataset == 'mnist' or args.dataset == 'fashion': num_pixels = 784 else: num_pixels = 3072 model, is_conv, means, stds = read_net(args.net, num_pixels, is_trained_with_pytorch) eran = ERAN(model) csvfile = open('../../code/datasets/{}_test.csv'.format(args.dataset), 'r') tests = csv.reader(csvfile, delimiter=',') total, attacked, standard_correct, tot_time = 0, 0, 0, 0 correct_box, correct_poly = 0, 0 cver_box, cver_poly = [], [] for i, test in enumerate(tests): if args.test_idx is not None and i != args.test_idx: continue attacks_file = os.path.join(args.data_dir, 'attack_{}.csv'.format(i)) if args.num_tests is not None and i >= args.num_tests: break print('Test {}:'.format(i)) #LRJ add current_test = i if args.dataset == 'mnist' or args.dataset == 'fashion': image = np.float64(test[1:len(test)]) else: if is_trained_with_pytorch: image = np.float64(test[1:len(test)]) else: image = np.float64(test[1:len(test)]) - 0.5 spec_lb = np.copy(image) spec_ub = np.copy(image) if (is_trained_with_pytorch): normalize(spec_lb, means, stds, args.dataset, is_conv) normalize(spec_ub, means, stds, args.dataset, is_conv) label, nn, nlb, nub = eran.analyze_box(spec_lb, spec_ub, 'deeppoly', args.timeout_lp, args.timeout_milp, args.use_area_heuristic) print('Label: ', label) if label != int(test[0]): print('Label {}, but true label is {}, skipping...'.format( label, int(test[0]))) print('Standard accuracy: {} percent'.format(standard_correct / float(i + 1) * 100)) continue else: standard_correct += 1 print('Standard accuracy: {} percent'.format(standard_correct / float(i + 1) * 100)) dim = n_rows * n_cols * n_channels ok_box, ok_poly = True, True k = args.num_params + 1 + 1 + dim attack_imgs, checked, attack_pass = [], [], 0 cex_found = False if args.attack: with open(attacks_file, 'r') as fin: lines = fin.readlines() for j in range(0, len(lines), args.num_params + 1): params = [ float(line[:-1]) for line in lines[j:j + args.num_params] ] tokens = lines[j + args.num_params].split(',') values = np.array(list(map(float, tokens))) attack_lb = values[::2] attack_ub = values[1::2] if is_trained_with_pytorch: normalize(attack_lb, means, stds, args.dataset, is_conv) normalize(attack_ub, means, stds, args.dataset, is_conv) else: attack_lb -= 0.5 attack_ub -= 0.5 attack_imgs.append((params, attack_lb, attack_ub)) checked.append(False) predict_label, _, _, _ = eran.analyze_box( attack_lb[:dim], attack_ub[:dim], 'deeppoly', args.timeout_lp, args.timeout_milp, args.use_area_heuristic, 0) if predict_label != int(test[0]): print('counter-example, params: ', params, ', predicted label: ', predict_label) cex_found = True break else: attack_pass += 1 print('tot attacks: ', len(attack_imgs)) specs_file = os.path.join(args.data_dir, '{}.csv'.format(i)) #LRJ add spec_lb_all = [] spec_ub_all = [] begtime = time.time() with open(specs_file, 'r') as fin: lines = fin.readlines() print('Number of lines: ', len(lines)) assert len(lines) % k == 0 spec_lb = np.zeros(args.num_params + dim) spec_ub = np.zeros(args.num_params + dim) expr_size = args.num_params lexpr_cst, uexpr_cst = [], [] lexpr_weights, uexpr_weights = [], [] lexpr_dim, uexpr_dim = [], [] ver_chunks_box, ver_chunks_poly, tot_chunks = 0, 0, 0 for i, line in enumerate(lines): if i % k < args.num_params: # read specs for the parameters values = np.array(list(map(float, line[:-1].split(' ')))) assert values.shape[0] == 2 param_idx = i % k spec_lb[dim + param_idx] = values[0] spec_ub[dim + param_idx] = values[1] if args.debug: print('parameter %d: [%.4f, %.4f]' % (param_idx, values[0], values[1])) elif i % k == args.num_params: # read interval bounds for image pixels values = np.array(list(map(float, line[:-1].split(',')))) #print(values) #print(len(values)) spec_lb[:dim] = values[::2] spec_ub[:dim] = values[1::2] # if args.debug: # show_ascii_spec(spec_lb, spec_ub) elif i % k < k - 1: # read polyhedra constraints for image pixels tokens = line[:-1].split(' ') assert len(tokens) == 2 + 2 * args.num_params + 1 bias_lower, weights_lower = float(tokens[0]), list( map(float, tokens[1:1 + args.num_params])) assert tokens[args.num_params + 1] == '|' bias_upper, weights_upper = float( tokens[args.num_params + 2]), list( map(float, tokens[3 + args.num_params:])) assert len(weights_lower) == args.num_params assert len(weights_upper) == args.num_params lexpr_cst.append(bias_lower) uexpr_cst.append(bias_upper) for j in range(args.num_params): lexpr_dim.append(dim + j) uexpr_dim.append(dim + j) lexpr_weights.append(weights_lower[j]) uexpr_weights.append(weights_upper[j]) else: assert (line == 'SPEC_FINISHED\n') for p_idx in range(args.num_params): lexpr_cst.append(spec_lb[dim + p_idx]) for l in range(args.num_params): lexpr_weights.append(0) lexpr_dim.append(dim + l) uexpr_cst.append(spec_ub[dim + p_idx]) for l in range(args.num_params): uexpr_weights.append(0) uexpr_dim.append(dim + l) #LRJ Add Data beform normalize if len(spec_lb_all): spec_lb_all = np.minimum(spec_lb_all, spec_lb[:dim]) else: spec_lb_all = list(spec_lb[:dim]) if len(spec_ub_all): spec_ub_all = np.maximum(spec_ub_all, spec_ub[:dim]) else: spec_ub_all = list(spec_ub[:dim]) #print(spec_lb_all) if (is_trained_with_pytorch): normalize(spec_lb[:dim], means, stds, args.dataset, is_conv) normalize(spec_ub[:dim], means, stds, args.dataset, is_conv) normalize_poly(args.num_params, lexpr_cst, lexpr_weights, lexpr_dim, uexpr_cst, uexpr_weights, uexpr_dim, means, stds, args.dataset) for attack_idx, (attack_params, attack_lb, attack_ub) in enumerate(attack_imgs): ok_attack = True for j in range(num_pixels): low, up = lexpr_cst[j], uexpr_cst[j] for idx in range(args.num_params): low += lexpr_weights[j * args.num_params + idx] * attack_params[idx] up += uexpr_weights[j * args.num_params + idx] * attack_params[idx] if low > attack_lb[j] + EPS or attack_ub[ j] > up + EPS: ok_attack = False if ok_attack: checked[attack_idx] = True # print('checked ', attack_idx) if args.debug: print('Running the analysis...') t_begin = time.time() perturbed_label_poly, _, _, _ = eran.analyze_box( spec_lb, spec_ub, 'deeppoly', args.timeout_lp, args.timeout_milp, args.use_area_heuristic, 0, lexpr_weights, lexpr_cst, lexpr_dim, uexpr_weights, uexpr_cst, uexpr_dim, expr_size) perturbed_label_box, _, _, _ = eran.analyze_box( spec_lb[:dim], spec_ub[:dim], 'deeppoly', args.timeout_lp, args.timeout_milp, args.use_area_heuristic, 0) t_end = time.time() #LRJ add normalized data below #print(spec_lb[:dim]) #if len(spec_lb_all): # spec_lb_all=np.minimum(spec_lb_all,spec_lb[:dim]) #else: # spec_lb_all=spec_lb[:dim] #if len(spec_ub_all): # spec_ub_all=np.maximum(spec_ub_all,spec_ub[:dim]) #else: # spec_ub_all=spec_ub[:dim] print('DeepG: ', perturbed_label_poly, '\tInterval: ', perturbed_label_box, '\tlabel: ', label, '[Time: %.4f]' % (t_end - t_begin)) tot_chunks += 1 if perturbed_label_box != label: ok_box = False else: ver_chunks_box += 1 if perturbed_label_poly != label: ok_poly = False else: ver_chunks_poly += 1 lexpr_cst, uexpr_cst = [], [] lexpr_weights, uexpr_weights = [], [] lexpr_dim, uexpr_dim = [], [] #LRJ add #print(len(spec_lb_all)) #print(len(spec_ub_all)) in_file = os.path.join(deepsymbol_input_folder, 'in_{}.in'.format(current_test)) fp = open(in_file, "w") for index in range(len(spec_lb_all)): fp.write( str(spec_lb_all[index]) + ' ' + str(spec_ub_all[index]) + '\n') fp.close() total += 1 if ok_box: correct_box += 1 if ok_poly: correct_poly += 1 if cex_found: assert (not ok_box) and (not ok_poly) attacked += 1 cver_poly.append(ver_chunks_poly / float(tot_chunks)) cver_box.append(ver_chunks_box / float(tot_chunks)) tot_time += time.time() - begtime print('Verified[box]: {}, Verified[poly]: {}, CEX found: {}'.format( ok_box, ok_poly, cex_found)) assert not cex_found or not ok_box, 'ERROR! Found counter-example, but image was verified with box!' assert not cex_found or not ok_poly, 'ERROR! Found counter-example, but image was verified with poly!' print('Attacks found: %.2f percent, %d/%d' % (100.0 * attacked / total, attacked, total)) print('[Box] Provably robust: %.2f percent, %d/%d' % (100.0 * correct_box / total, correct_box, total)) print('[Poly] Provably robust: %.2f percent, %d/%d' % (100.0 * correct_poly / total, correct_poly, total)) print('Empirically robust: %.2f percent, %d/%d' % (100.0 * (total - attacked) / total, total - attacked, total)) print('[Box] Average chunks verified: %.2f percent' % (100.0 * np.mean(cver_box))) print('[Poly] Average chunks verified: %.2f percent' % (100.0 * np.mean(cver_poly))) print('Average time: ', tot_time / total) print( 'Verified:%d Box Robust:%d Poly Robust:%d Box Robust Percentage:%.2f Poly Robust Percentage:%.2f' % (total, correct_box, correct_poly, 100.0 * correct_box / total, 100.0 * correct_poly / total))
return (y * std_deviations) + means args = { "complete": False, "timeout_lp": 1, "timeout_milp": 1, "use_area_heuristic": True, } netname = sys.argv[1] filename, file_extension = os.path.splitext(netname) num_pixels = 5 model, is_conv, means, stds = read_net(netname, num_pixels, False) eran = ERAN(model) def extract_from_expr(expr, coeffs=5): coefficients = [] for i in range(coeffs): coeff = elina_linexpr0_coeffref(expr, i) assert coeff.contents.discr == 1 interval = coeff.contents.val.interval.contents assert interval.inf.contents.discr == 0 assert interval.sup.contents.discr == 0 inf = interval.inf.contents.val.dbl sup = interval.sup.contents.val.dbl coefficients.append([inf, sup]) cst = elina_linexpr0_cstref(expr) assert cst.contents.discr == 1
def _acasxu_recursive(specLB, specUB, model, eran: ERAN, constraints, failed_already, max_depth=10, depth=0, permitted_depth_extensions=3, depth_extension=0, domain="deeppoly", timeout_lp=1, timeout_milp=1, use_default_heuristic=True, complete=True) \ -> Tuple[bool, Optional[Sequence[Tuple[np.ndarray, Tuple[int, int, float]]]]]: # Returns whether the constraint is violated, and potentially counterexamples as witness of violation. # The counterexamples are returned together with the atomic comparison constraint they violate. hold, nn, nlb, nub, _, _ = \ eran.analyze_box(specLB, specUB, domain, timeout_lp, timeout_milp, use_default_heuristic, constraints) if hold: return True, [] elif depth >= max_depth: if failed_already.value and complete: # failed_already.value means "not falsified already" verified_flag, adv_examples, adv_values, adv_violated_constraints = \ verify_network_with_milp(nn, specLB, specUB, nlb, nub, constraints) xs = [] if verified_flag: return True, [] else: if adv_examples is not None: for adv_image, violated_constraint, adv_value in \ zip(adv_examples, adv_violated_constraints, adv_values): # counterexamples found with milp may be improper counterexamples # for constraints with multiple or-clauses, for example the or clauses # are treated individually and returned counterexamples violate individual # or-clauses, but they do not necessarily violate the whole or-term. hold, _, nlb, nub, _, x = eran.analyze_box( adv_image, adv_image, domain, timeout_lp, timeout_milp, use_default_heuristic, constraints) # print(f'hold={hold}, adv_image={adv_image}') if not hold: info( f"property violated at {adv_image} with violation {adv_value} output_score {nlb[-1]}" ) failed_already.value = 0 # break # do not break, try to find many counterexamples else: # spurious counterexample info( f"spurious counterexample found: {adv_image} with violation {adv_value} " f"output_score {nlb[-1]}") # for spurious counterexamples we continue search, maybe we will still find a proper cx xs.append((adv_image, violated_constraint, adv_value)) if len(xs) > 0: # counterexample found return False, xs # if verified_flag was false, but no concrete counterexamples were found # continue with bounds splitting (perform depth_extension) unless the number # of permitted depth extensions was also reached if depth_extension >= permitted_depth_extensions: return False, None # debug(f"Extending Recursion Depth ({depth_extension+1}. time)") depth_extension += 1 depth = 0 else: return False, None # grads = _estimate_grads(specLB, specUB, model, input_shape=eran.input_shape) # grads + small epsilon so if gradient estimation becomes 0 it will divide the biggest interval. # smears = np.multiply(grads + 0.00001, [u-l for u, l in zip(specUB, specLB)]) nn.set_last_weights(constraints) grads_lower, grads_upper = nn.back_propagate_gradient(nlb, nub) smears = [ max(-grad_l, grad_u) * (u - l) for grad_l, grad_u, l, u in zip( grads_lower, grads_upper, specLB, specUB) ] index = np.argmax(smears) m = (specLB[index] + specUB[index]) / 2 result = failed_already.value xs = [] if result: result1, xs1 = _acasxu_recursive( specLB, [ub if i != index else m for i, ub in enumerate(specUB)], model, eran, constraints, failed_already, max_depth, depth + 1, permitted_depth_extensions, depth_extension, domain, timeout_lp, timeout_milp, use_default_heuristic, complete) result = result and result1 if xs1 is not None: xs.extend(xs1) if result: result2, xs2 = _acasxu_recursive( [lb if i != index else m for i, lb in enumerate(specLB)], specUB, model, eran, constraints, failed_already, max_depth, depth + 1, permitted_depth_extensions, depth_extension, domain, timeout_lp, timeout_milp, use_default_heuristic, complete) result = result and result2 if xs2 is not None: xs.extend(xs2) return result, xs
i += 1 if label == corr_label: prec += 1 if i % 100 == 0: print( prec , '/', i ) exit() ''' from clever_wolf import * import sys sys.path.insert(0, '../ELINA/python_interface/') from eran import ERAN from config import config config.dyn_krelu = False config.numproc_krelu = 1 eran = ERAN(cut_model.tf_output, is_onnx=False) imgLB = np.copy(img) imgUB = np.copy(img) label, nn, nlb, nub = eran.analyze_box(imgLB, imgUB, 'deepzono', 1, 1, True) assert label == cut_model.y_true cut_model.orig_image = image.copy() if dataset == 'acasxu' or dataset == 'mortgage': corr_label = cut_model.y_true # Create specLB/UB if dataset == 'mnist': specLB = np.clip(image - epsilon, 0, 1) specUB = np.clip(image + epsilon, 0, 1) elif dataset == 'mortgage':
def verify_plain(network_file: str, means: np.ndarray, stds: np.ndarray, input_box: List[Tuple[np.ndarray, np.ndarray]], output_constraints: List[List[Tuple[int, int, float]]], timeout_lp=1, timeout_milp=1, use_default_heuristic=True, use_milp=True, progress_bar=True) \ -> Optional[Sequence[Tuple[np.ndarray, Tuple[int, int, float], Optional[float]]]]: """ Verifies a neural network. This method also supports convolutional networks, but has much lower precision than ``verify_acasxu_style``, because it does not split the input bounds. Returns None if an no counterexamples could be found and the network could not be verified. Returns an empty list if the network could be verified. Returns a list of inputs for which the network violates the constraints if the network could not be verified. """ domain = "deeppoly" model, is_conv = read_onnx_net(network_file) eran = ERAN(model, is_onnx=True) specLB = [interval[0] for interval in input_box] specUB = [interval[1] for interval in input_box] _normalize(specLB, means, stds) _normalize(specUB, means, stds) counterexample_list = [] # adex_holds stores whether x_adex (below) is actually a counterexample # if adex_holds is True, then x_adex is a spurious counterexample adex_holds = True with tqdm(total=3, disable=not progress_bar) as progress_bar: verified_flag, nn, nlb, nub, _, x_adex = eran.analyze_box( specLB, specUB, domain, timeout_lp, timeout_milp, use_default_heuristic, output_constraints) progress_bar.update() if not verified_flag and x_adex is not None: adex_holds, _, _, _, _, _ = eran.analyze_box( x_adex, x_adex, "deeppoly", timeout_lp, timeout_milp, use_default_heuristic, output_constraints) if not adex_holds: verified_flag = False # we need to undo the input normalisation, that was applied to the counterexamples counterexample_list.append((np.array(x_adex) * stds + means, output_constraints, None)) progress_bar.update() # try to find a counterexample with milp solving if not verified_flag and adex_holds and use_milp: verified_flag, adv_examples, adv_values, adv_violated_constraints = \ verify_network_with_milp(nn, specLB, specUB, nlb, nub, output_constraints) if not verified_flag: # compare with milp solving code above if adv_examples is not None: for adv_image, violated_constraint, adv_value in \ zip(adv_examples, adv_violated_constraints, adv_values): hold, _, nlb, nub, _, x = eran.analyze_box( adv_image, adv_image, domain, timeout_lp, timeout_milp, use_default_heuristic, output_constraints) if not hold: info( f"property violated at {adv_image} with violation {adv_value} output_score {nlb[-1]}" ) else: info( f"spurious counterexample found: {adv_image} with violation {adv_value} " f"output_score {nlb[-1]}") counterexample_list.append( (np.array(adv_image) * stds + means, violated_constraint, adv_value)) progress_bar.update() if not verified_flag and len(counterexample_list) > 0: info( f"Property not verified with counterexamples: {counterexample_list}." ) return counterexample_list elif not verified_flag: info(f"Property not verified without counterexamples.") # raise RuntimeError("Property disproven, but no counterexample found.") return None else: info(f"Property verified.") return []
print("netname ", netname, " specnumber ", specnumber, " domain ", domain, " dataset ", dataset, "args complete ", args.complete, " complete ", complete, " timeout_lp ", args.timeout_lp) else: print("netname ", netname, " epsilon ", epsilon, " domain ", domain, " dataset ", dataset, "args complete ", args.complete, " complete ", complete, " timeout_lp ", args.timeout_lp) if (is_saved_tf_model): netfolder = os.path.dirname(netname) tf.logging.set_verbosity(tf.logging.ERROR) sess = tf.Session() saver = tf.train.import_meta_graph(netname) saver.restore(sess, tf.train.latest_checkpoint(netfolder + '/')) eran = ERAN(sess.graph.get_tensor_by_name('logits:0'), sess) else: if (dataset == 'mnist'): num_pixels = 784 elif (dataset == 'cifar10'): num_pixels = 3072 else: num_pixels = 5 model, is_conv, means, stds = read_net(netname, num_pixels, is_trained_with_pytorch) eran = ERAN(model) correctly_classified_images = 0 verified_images = 0 total_images = 0
saver = tf.train.import_meta_graph(netname) saver.restore(sess, tf.train.latest_checkpoint(netfolder+'/')) else: with tf.gfile.GFile(netname, "rb") as f: graph_def = tf.GraphDef() graph_def.ParseFromString(f.read()) sess.graph.as_default() tf.graph_util.import_graph_def(graph_def, name='') ops = sess.graph.get_operations() last_layer_index = -1 while ops[last_layer_index].type in non_layer_operation_types: last_layer_index -= 1 out_tensor = sess.graph.get_tensor_by_name(ops[last_layer_index].name + ':0') try: eran = ERAN(out_tensor, sess) except Exception as e: tested_file.write(', '.join([dataset, network, 'ERAN parse error message: ' + str(e)]) + '\n') tested_file.flush() continue else: if(dataset=='mnist'): num_pixels = 784 elif (dataset=='cifar10'): num_pixels = 3072 elif(dataset=='acasxu'): num_pixels = 5 if is_onnx: model, is_conv = read_onnx_net(netname) # this is a hack and should be done nicer
def __init__(self, model): self.eran = ERAN(model=model, is_onnx=True)
class EranVerifier: """Wrapper object for the ERAN verifier to have a clean interface with proper types.""" __TIMEOUT_LP = 1_000_000 __TIMEOUT_MILP = 1_000_000 def __init__(self, model): self.eran = ERAN(model=model, is_onnx=True) def analyze_classification_box(self, bounds: Interval): (dominant_class, _, nlb, nub, _) = self.eran.analyze_box(specLB=bounds.lower_bound, specUB=bounds.upper_bound, domain='deeppoly', timeout_lp=EranVerifier.__TIMEOUT_LP, timeout_milp=EranVerifier.__TIMEOUT_MILP, use_default_heuristic=True, testing=True) return dominant_class, nlb, nub def analyze_classification_linear(self, bounds: Interval, constraints: LinearBounds, params: List[Interval]): res = self.__eran_resources(bounds, constraints, params) (dominant_class, _, nlb, nub, _) = self.eran.analyze_box(specLB=res.lower_bound, specUB=res.upper_bound, domain='deeppoly', timeout_lp=EranVerifier.__TIMEOUT_LP, timeout_milp=EranVerifier.__TIMEOUT_MILP, lexpr_weights=res.lexpr_weights, lexpr_cst=res.lexpr_cst, lexpr_dim=res.lexpr_dim, uexpr_weights=res.uexpr_weights, uexpr_cst=res.uexpr_cst, uexpr_dim=res.uexpr_dim, expr_size=res.expr_size, use_default_heuristic=True, testing=True) return dominant_class, nlb, nub def analyze_segmentation_box(self, bounds: Interval, correct_labels: np.ndarray, valid_labels: np.ndarray, num_total_classes: int): (dominant_classes, nlb, nub) = EranVerifier.__analyze_segmentation_eran( eran=self.eran, true_labels=correct_labels, valid_classes=valid_labels, num_total_classes=num_total_classes, specLB=bounds.lower_bound, specUB=bounds.upper_bound, domain='deeppoly', timeout_lp=EranVerifier.__TIMEOUT_LP, timeout_milp=EranVerifier.__TIMEOUT_MILP, use_default_heuristic=True, testing=True) return dominant_classes, nlb, nub def analyze_segmentation_linear(self, bounds: Interval, constraints: LinearBounds, params: List[Interval], correct_labels: np.ndarray, valid_labels: np.ndarray, num_total_classes: int): res = self.__eran_resources(bounds, constraints, params) (dominant_classes, nlb, nub) = EranVerifier.__analyze_segmentation_eran( eran=self.eran, true_labels=correct_labels, valid_classes=valid_labels, num_total_classes=num_total_classes, specLB=res.lower_bound, specUB=res.upper_bound, domain='deeppoly', timeout_lp=EranVerifier.__TIMEOUT_LP, timeout_milp=EranVerifier.__TIMEOUT_MILP, lexpr_weights=res.lexpr_weights, lexpr_cst=res.lexpr_cst, lexpr_dim=res.lexpr_dim, uexpr_weights=res.uexpr_weights, uexpr_cst=res.uexpr_cst, uexpr_dim=res.uexpr_dim, expr_size=res.expr_size, use_default_heuristic=True, testing=True) return dominant_classes, nlb, nub # adapted from ERAN/tf_verify/ERAN to support segmentation @staticmethod def __analyze_segmentation_eran(eran: ERAN, true_labels, valid_classes, num_total_classes, specLB, specUB, domain, timeout_lp, timeout_milp, use_default_heuristic, lexpr_weights=None, lexpr_cst=None, lexpr_dim=None, uexpr_weights=None, uexpr_cst=None, uexpr_dim=None, expr_size=0, testing=False, prop=-1, spatial_constraints=None): assert domain == "deeppoly", "domain isn't valid, must be 'deeppoly'" specLB = np.reshape(specLB, (-1, )) specUB = np.reshape(specUB, (-1, )) eran.nn = layers() eran.nn.specLB = specLB eran.nn.specUB = specUB execute_list, output_info = eran.optimizer.get_deeppoly( eran.nn, specLB, specUB, lexpr_weights, lexpr_cst, lexpr_dim, uexpr_weights, uexpr_cst, uexpr_dim, expr_size, spatial_constraints) analyzer = Analyzer(execute_list, eran.nn, domain, timeout_lp, timeout_milp, None, use_default_heuristic, -1, prop, testing) dominant_classes, nlb, nub = EranVerifier.__analyze_segmentation_analyzer( analyzer, true_labels, valid_classes, num_total_classes) return dominant_classes, nlb, nub # adapted from ERAN/tf_verify/Analyzer to support segmentation @staticmethod def __analyze_segmentation_analyzer(analyzer: Analyzer, true_labels, valid_classes, num_classes: int): element, nlb, nub = analyzer.get_abstract0() number_points = len(true_labels) dominant_classes = [] for n in range(number_points): dominant_class = -1 label_failed = [] candidate_labels = valid_classes true_label = true_labels[n] certified = True for candidate_label in candidate_labels: point_offset = n * num_classes if candidate_label != true_label and not analyzer.is_greater( analyzer.man, element, point_offset + true_label, point_offset + candidate_label, analyzer.use_default_heuristic): certified = False label_failed.append(candidate_label) break if certified: dominant_class = true_label dominant_classes.append(dominant_class) elina_abstract0_free(analyzer.man, element) return dominant_classes, nlb, nub @staticmethod def __eran_resources(bounds: Interval, constraints: LinearBounds, params: List[Interval]) -> EranResources: res = EranResources() res.lower_bound = np.append(bounds.lower_bound, [p.lower_bound for p in params]) res.upper_bound = np.append(bounds.upper_bound, [p.upper_bound for p in params]) expr_size = len(params) res.expr_size = expr_size num_indices = len(res.lower_bound.flatten()) param_indices = np.arange(num_indices - expr_size, num_indices) res.lexpr_weights = np.append(constraints.lower_slope.flatten(), np.zeros(expr_size * expr_size)) res.uexpr_weights = np.append(constraints.upper_slope.flatten(), np.zeros(expr_size * expr_size)) res.lexpr_cst = np.append(constraints.lower_offset.flatten(), [p.lower_bound for p in params]) res.uexpr_cst = np.append(constraints.upper_offset.flatten(), [p.upper_bound for p in params]) res.lexpr_dim = np.repeat([param_indices], num_indices, axis=0).flatten() res.uexpr_dim = np.repeat([param_indices], num_indices, axis=0).flatten() assert len(res.lower_bound) == num_indices assert len(res.upper_bound) == num_indices assert len(res.lexpr_weights) == num_indices * expr_size assert len(res.uexpr_weights) == num_indices * expr_size assert len(res.lexpr_cst) == num_indices assert len(res.uexpr_cst) == num_indices assert len(res.lexpr_dim) == num_indices * expr_size assert len(res.uexpr_dim) == num_indices * expr_size return res
exit(1) is_conv = False mean = 0 std = 0 if(is_saved_tf_model): netfolder = os.path.dirname(netname) tf.logging.set_verbosity(tf.logging.ERROR) sess = tf.Session() saver = tf.train.import_meta_graph(netname) saver.restore(sess, tf.train.latest_checkpoint(netfolder+'/')) eran = ERAN(sess.graph.get_tensor_by_name('logits:0'), sess) else: if(dataset=='mnist'): num_pixels = 784 else: num_pixels = 3072 model, is_conv, means, stds = read_net(netname, num_pixels, is_trained_with_pytorch) eran = ERAN(model) correctly_classified_images = 0 verified_images = 0 total_images = 0 def normalize(image, means, stds):
sess = tf.Session() if is_saved_tf_model: saver = tf.train.import_meta_graph(netname) saver.restore(sess, tf.train.latest_checkpoint(netfolder + '/')) else: with tf.gfile.GFile(netname, "rb") as f: graph_def = tf.GraphDef() graph_def.ParseFromString(f.read()) sess.graph.as_default() tf.graph_util.import_graph_def(graph_def, name='') ops = sess.graph.get_operations() last_layer_index = -1 while ops[last_layer_index].type in non_layer_operation_types: last_layer_index -= 1 eran = ERAN( sess.graph.get_tensor_by_name(ops[last_layer_index].name + ':0'), sess) else: if (zonotope_bool == True): num_pixels = len(zonotope) elif (dataset == 'mnist'): num_pixels = 784 elif (dataset == 'cifar10'): num_pixels = 3072 elif (dataset == 'acasxu'): num_pixels = 5 if is_onnx: model, _ = read_onnx_net(netname) # this is to have different defaults for mnist and cifar10 else: model, _, means, stds = read_tensorflow_net(netname, num_pixels,
def clever_wolf(nn, cut_model, y_true, y_tar, specLB, specUB, domain, args): clever_start_time = time.time() if cut_model.y_tar == None: data, lb, ub = generate_initial_region_PGD(y_tar, 250) reset_pool((specLB, specUB)) update_target_pool((y_true, y_tar)) cut_model.update_target(y_true, y_tar) cut_model.reset_model(specLB, specUB) succ_attacks = data.shape[0] all_attacks = 250 if not args.nowolf: samples = cut_model.sample_poly_under(250) pos_ex = sample_wolf_attacks(samples, 'pos_brent') succ_attacks += pos_ex.shape[0] all_attacks += 250 print('Target', y_tar, succ_attacks, '/', all_attacks) if succ_attacks > 0: data, lb, ub = generate_initial_region_PGD(y_tar, 5000) reset_pool((specLB, specUB)) update_target_pool((y_true, y_tar)) cut_model.update_target(y_true, y_tar) cut_model.reset_model(specLB, specUB) if not args.nowolf: samples = cut_model.sample_poly_under(5000) pos_ex = sample_wolf_attacks(samples, 'pos_brent') if not pos_ex.shape[0] == 0: data = np.concatenate((data, pos_ex)) lb = np.min(data, axis=0) ub = np.max(data, axis=0) cut_model.update_bounds(lb, ub) cut_model.set_data(data) update_pool((lb, ub)) else: return False s = time.time() config.dyn_krelu = False config.use_3relu = False config.use_2relu = False config.numproc_krelu = 24 eran = ERAN(cut_model.tf_output, is_onnx=False) label, nn, nlb, nub = eran.analyze_box(lb, ub, 'deeppoly', 1, 1, True) print('Label:', label, 'Time:', time.time() - s, 's') if label == -1: cut_model.nlb = [np.array(lb) for lb in nlb] cut_model.nub = [np.array(ub) for ub in nub] lb, ub = cut_model.overapprox_box() cut_model.nlb.insert(0, lb) cut_model.nub.insert(0, ub) else: cut_model.save(model_name) print('Verified, time:', int(time.time() - clever_start_time)) return True print('Init model') if args.obox_approx: cut_model.approx_obox = True process = psutil.Process(os.getpid()) start_lp_sampling = args.nowolf method = None res = None hist = History(cut_model.input_size, cut_hist_size=5, update_hist_every=2) lp_params = (nn, domain, y_tar) wolf_params = [1000] for cut in range(args.max_cuts): sys.stdout.flush() print_vol(cut_model) gc.collect() print('\nCut:', cut, ', Time:', int(time.time() - clever_start_time), 'sec,', 'Target:', y_tar, ',X_size', cut_model.data_size, ',Memory:', process.memory_info().rss / (1024 * 1024), '\n') if cut % args.save_every == 0: cut_model.save(model_name) if not start_lp_sampling and cut % args.choose_criterea_every == 0: method = choose_method(cut_model, hist, lp_params, wolf_params) print('Chosen new method:', method) if method == None: cut_model.save(model_name) print('Verified, time:', int(time.time() - clever_start_time)) return True if not start_lp_sampling and method == 'Wolf': res = wolf_cut(cut_model, hist, *wolf_params) if start_lp_sampling or res == False or method == 'LP': if not start_lp_sampling and not method == 'LP': start_lp_sampling = start_lp_sampling or res == False verified, _ = lp_cut(cut_model, hist, *lp_params) if verified: cut_model.save(model_name) print('Verified, time:', int(time.time() - clever_start_time)) return True cut_model.shrink_poly(nn, domain, y_tar) cut_model.save(model_name) print_vol(cut_model) print('Verified, time:', int(time.time() - clever_start_time)) return True
sess = tf.Session() if is_saved_tf_model: saver = tf.train.import_meta_graph(netname) saver.restore(sess, tf.train.latest_checkpoint(netfolder + '/')) else: with tf.gfile.GFile(netname, "rb") as f: graph_def = tf.GraphDef() graph_def.ParseFromString(f.read()) sess.graph.as_default() tf.graph_util.import_graph_def(graph_def, name='') ops = sess.graph.get_operations() last_layer_index = -1 while ops[last_layer_index].type in non_layer_operation_types: last_layer_index -= 1 eran = ERAN( sess.graph.get_tensor_by_name(ops[last_layer_index].name + ':0'), sess) else: if (zonotope_bool == True): num_pixels = len(zonotope) elif (dataset == 'mnist'): num_pixels = 784 elif (dataset == 'cifar10'): num_pixels = 3072 elif (dataset == 'acasxu'): num_pixels = 5 if is_onnx: model, is_conv = read_onnx_net(netname) # this is to have different defaults for mnist and cifar10 else: model, is_conv, means, stds = read_tensorflow_net(
tf.train.latest_checkpoint(netfolder + '/')) else: with tf.gfile.GFile(netname, "rb") as f: graph_def = tf.GraphDef() graph_def.ParseFromString(f.read()) sess.graph.as_default() tf.graph_util.import_graph_def(graph_def, name='') ops = sess.graph.get_operations() last_layer_index = -1 while ops[last_layer_index].type in non_layer_operation_types: last_layer_index -= 1 out_tensor = sess.graph.get_tensor_by_name( ops[last_layer_index].name + ':0') try: eran = ERAN(out_tensor, sess) except Exception as e: tested_file.write(', '.join([ dataset, network, 'ERAN parse error trace: ' + traceback.format_exc() ]) + '\n\n\n') tested_file.flush() continue else: if (dataset == 'mnist'): num_pixels = 784 elif (dataset == 'cifar10'): num_pixels = 3072 elif (dataset == 'acasxu'): num_pixels = 5
sess = tf.Session() if is_saved_tf_model: saver = tf.train.import_meta_graph(netname) saver.restore(sess, tf.train.latest_checkpoint(netfolder+'/')) else: with tf.gfile.GFile(netname, "rb") as f: graph_def = tf.GraphDef() graph_def.ParseFromString(f.read()) sess.graph.as_default() tf.graph_util.import_graph_def(graph_def, name='') ops = sess.graph.get_operations() last_layer_index = -1 while ops[last_layer_index].type in non_layer_operation_types: last_layer_index -= 1 model = sess.graph.get_tensor_by_name(ops[last_layer_index].name + ':0') eran = ERAN(model, sess) else: if(zonotope_bool==True): num_pixels = len(zonotope) elif(dataset=='mnist'): num_pixels = 784 elif (dataset=='cifar10'): num_pixels = 3072 elif(dataset=='acasxu'): num_pixels = 5 if is_onnx: model, is_conv = read_onnx_net(netname) else: model, is_conv, means, stds = read_tensorflow_net(netname, num_pixels, is_trained_with_pytorch) eran = ERAN(model, is_onnx=is_onnx)
def clever_wolf(nn, cut_model, y_true, y_tar, specLB, specUB, domain, args, pgd_args1, pgd_args2): try: delattr(cut_model, 'shrink_box_best') except: pass clever_start_time = time.time() if cut_model.y_tar == None: reset_pool((specLB, specUB)) update_target_pool((y_true, y_tar)) data, _, lb, ub = generate_initial_region_PGD(250, *pgd_args1) cut_model.update_target(y_true, y_tar) cut_model.reset_model(specLB, specUB) succ_attacks = data.shape[0] all_attacks = 250 if not args.nowolf: samples = cut_model.sample_poly_under(250) pos_ex = sample_wolf_attacks(samples, 'pos_brent') succ_attacks += pos_ex.shape[0] all_attacks += 250 data2, _, lb2, ub2 = generate_initial_region_PGD(250, *pgd_args2) succ_attacks += data2.shape[0] all_attacks += 250 print('Target', y_tar, succ_attacks, '/', all_attacks) if succ_attacks > 0: reset_pool((specLB, specUB)) update_target_pool((y_true, y_tar)) cut_model.update_target(y_true, y_tar) cut_model.reset_model(specLB, specUB) data, bounds, lb, ub = generate_initial_region_PGD( 2500, *pgd_args1) data2, bounds2, lb2, ub2 = generate_initial_region_PGD( 2500, *pgd_args2) if lb is None and lb2 is None: data = np.zeros((0, cut_model.input_size)) bounds = np.zeros(0) elif lb is None: data = data2 bounds = bounds2 elif not lb2 is None: data = np.concatenate((data, data2)) bounds = np.concatenate((bounds, bounds2)) if not args.nowolf: samples = cut_model.sample_poly_under(5000) pos_ex = sample_wolf_attacks(samples, 'pos_brent') if not pos_ex.shape[0] == 0: data = np.concatenate((data, pos_ex)) if data.shape[0] == 0: return False lb = np.min(data, axis=0) ub = np.max(data, axis=0) cut_model.update_bounds(lb, ub) cut_model.set_data(data) cut_model.bounds = bounds update_pool((lb, ub)) else: return False s = time.time() config.dyn_krelu = False config.use_3relu = False config.use_2relu = False config.numproc_krelu = 24 norm_ser = dill.dumps(normalize) denorm_ser = dill.dumps(denormalize) cut_model.norm_func = norm_ser cut_model.denorm_func = denorm_ser cut_model.dataset = dataset cut_model.is_conv = is_conv cut_model.norm_args = (means, stds) lb, ub = cut_model.overapprox_box() cut_model.orig_over_lb = lb cut_model.orig_over_ub = ub eran = ERAN(cut_model.tf_output, is_onnx=False) label, nn, nlb, nub = eran.analyze_box(lb, ub, 'deeppoly', 1, 1, True) print('Label:', label, 'Time:', time.time() - s, 's') if label == -1: cut_model.nlb = [np.array(lb) for lb in nlb] cut_model.nub = [np.array(ub) for ub in nub] lb, ub = cut_model.overapprox_box() cut_model.nlb.insert(0, lb) cut_model.nub.insert(0, ub) else: cut_model.shrink_box_best = cut_model.overapprox_box() cut_model.save(model_name) print('Verified, time:', int(time.time() - clever_start_time)) print_vol(cut_model) return True print('Init model') if args.obox_approx: cut_model.approx_obox = True process = psutil.Process(os.getpid()) start_lp_sampling = args.nowolf method = None res = None lp_params = (nn, domain, y_tar) wolf_params = [1000] shrink = cut_model.copy() if 'shrink_box_best' in dir(shrink): del shrink.shrink_box_best print_vol(cut_model) sys.stdout.flush() lbs, ubs, lb_max, ub_max = shrink.create_underapprox_box_lp( y_tar, shrink_to=args.obox_init, baseline=args.baseline) if args.baseline: print('\nBaseline, Time:', int(time.time() - clever_start_time), 'sec,', 'Target:', y_tar, '\n') if not lb_max is None: data = cut_model.data cut_model.obox = None cut_model.ubox = None cut_model.update_target(y_true, y_tar) cut_model.reset_model(lb_max, ub_max) cut_model.update_bounds(lb_max, ub_max) cut_model.nlb = [np.array(lb) for lb in nlb] cut_model.nub = [np.array(ub) for ub in nub] cut_model.nlb.insert(0, lb_max) cut_model.nub.insert(0, ub_max) cut_model.data = data if not lbs is None: cut_model.shrink_box_best = (np.array(lbs), np.array(ubs)) else: print('Failed to converge') return False print_vol(cut_model) if args.baseline: cut_model.save(model_name, baseline=True) return True cut_model.p_c = 0.9 for cut in range(50): cut_model.it_cut = cut cut_model.p_c *= 0.95 if cut >= args.max_cuts: cut_model.p_c = 0 sys.stdout.flush() ''' if dataset == 'mortgage': project = np.array( [ False ] * cut_model.input_size ) project[ bounds_keys ] = True W, lb, ub = cut_model.denorm_W( project, means, stds ) lb_sh, ub_sh = cut_model.shrink_box_best[0].copy(), cut_model.shrink_box_best[1].copy() denormalize( lb_sh, dataset, means, stds, False ) denormalize( ub_sh, dataset, means, stds, False ) lb_sh = lb_sh[bounds_keys] ub_sh = ub_sh[bounds_keys] center = None pos_ex = None neg_ex = None v2 = draw2d_region( W, lb, ub, lb_sh, ub_sh, model_name + '_reg' + str(cut) + '.png', ( bounds_lb, bounds_ub ), center=center, pos_ex=pos_ex, neg_ex=neg_ex, draw_hp=W.shape[0] - 1 ) print( 'Volume:', v2 ) ''' gc.collect() print('\nCut:', cut, ', Time:', int(time.time() - clever_start_time), 'sec,', 'Target:', y_tar, ',Memory:', process.memory_info().rss / (1024 * 1024), '\n') verified, _ = lp_cut(cut_model, *lp_params) if verified: if dataset == 'mortgage': project = np.array([False] * cut_model.input_size) project[bounds_keys] = True W, lb, ub = cut_model.denorm_W(project, means, stds) lb_sh, ub_sh = cut_model.shrink_box_best[0].copy( ), cut_model.shrink_box_best[1].copy() denormalize(lb_sh, dataset, means, stds, False) denormalize(ub_sh, dataset, means, stds, False) lb_sh = lb_sh[bounds_keys] ub_sh = ub_sh[bounds_keys] center = None pos_ex = None neg_ex = None v2 = draw2d_region(W, lb, ub, lb_sh, ub_sh, model_name + '_reg' + str(cut + 1) + '.png', (bounds_lb, bounds_ub), center=center, pos_ex=pos_ex, neg_ex=neg_ex, draw_hp=W.shape[0] - 1) print('Volume:', v2) cut_model.save(model_name) print('Verified, time:', int(time.time() - clever_start_time)) print_vol(cut_model) return True print('Failed to converge') return False
def main(): """Runs the ERAN analysis on the pendulum_continuous model. This happens in a few steps: 1. ERAN doesn't specifically support HardTanh layers, so we translate the HardTanh into an equivalent set of ReLU layers using convert_htanh(). 2. We then read the controller network into an ERAN model. 3. We use ERAN/DeepPoly to extract an abstract value describing the network's behavior over the initial set. Module float imprecision handling, this abstract value is basically two affine transform A and B such that Ax <= f(x) <= Bx for all x in the initial set. 4. We compute Ax and Bx for a particular point (0.35, 0.35) right on the edge of the initial set, and show that the range determined by DeepPoly (even after applying a concrete HardTanh at the end) is wide enough to mark that point as unsafe even on the first iteration. """ # (1) Translate the model into an equivalent one without HardTanh. with_htanh_filename = sys.argv[1] no_htanh_filename = "/ovol/pendulum_continuous.no_htanh.eran" convert_htanh(with_htanh_filename, no_htanh_filename) # (2) Read it into ERAN. num_pixels = 2 model, _, _, _ = read_net(no_htanh_filename, num_pixels, False) eran = ERAN(model) # (3) Extract an abstract value over the initial set. # (3a) Load model and init set into ERAN. initLB = np.array([-0.35, -0.35]) initUB = np.array([0.35, 0.35]) nn = layers() nn.specLB = initLB nn.specUB = initUB execute_list = eran.optimizer.get_deeppoly(initLB, initUB) # NOTE: 9 is just a placeholder specnumber to tell it we're using # ACAS. analyzer = Analyzer(execute_list, nn, "deeppoly", args["timeout_lp"], args["timeout_milp"], 9, args["use_area_heuristic"]) # (3b) Perform the analysis and extract the abstract values. element, _, _ = analyzer.get_abstract0() lexpr = get_lexpr_for_output_neuron(analyzer.man, element, 0) uexpr = get_uexpr_for_output_neuron(analyzer.man, element, 0) lexpr = np.array(extract_from_expr(lexpr)) uexpr = np.array(extract_from_expr(uexpr)) # (3c) Extract the output range for initLB based on the abstract value. lower_bound, upper_bound = compute_output_range(initLB, lexpr, uexpr) # Apply extra knowledge that -1 <= lower_bound <= upper_bound <= 1. lower_bound = max(lower_bound, -1.) upper_bound = min(upper_bound, 1.) post_lower, post_upper = post_bounds(initLB, lower_bound, upper_bound) post_lower, post_upper = post_lower.flatten(), post_upper.flatten() lower_safe = np.min(post_lower) >= -0.35 upper_safe = np.max(post_upper) <= 0.35 is_safe = lower_safe and upper_safe print("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~") if not is_safe: print("ERAN reported initLB unsafe after the first step.") else: print("Something changed; ERAN used to report initLB unsafe, but now" "it says it's safe.") print("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~") elina_abstract0_free(analyzer.man, element)