def MIM(model, sess, epsilon, num_steps, data=MNIST()): image_size = data.test_data.shape[1] num_channels = data.test_data.shape[3] num_labels = data.test_labels.shape[1] shape = (None, image_size, image_size, num_channels) model.x_input = tf.placeholder(tf.float32, shape) model.y_input = tf.placeholder(tf.float32, [None, num_labels]) pre_softmax = model(model.x_input) y_loss = tf.nn.softmax_cross_entropy_with_logits(labels=model.y_input, logits=pre_softmax) model.xent = tf.reduce_sum(y_loss) attack = Mim.MimAttack(model, epsilon, num_steps) if data.dataset == "cifar": sets = np.split(data.test_data, 2) label_sets = np.split(data.test_labels, 2) p1 = attack.perturb(sets[0], label_sets[0], sess) p2 = attack.perturb(sets[1], label_sets[1], sess) return np.concatenate((p1, p2), axis=0) return attack.perturb(np.array(data.test_data), np.array(data.test_labels), sess)
def get_accuracy(file_name, sess, epsilon, num_steps, step_size, data=MNIST()): model = load_model(file_name, custom_objects={'fn': loss, 'tf': tf, 'atan': tf.math.atan}) start_time = time.time() adversaries = PGD(model, sess, epsilon, num_steps, step_size, data) predictions = model.predict(adversaries) accuracy = np.mean(np.equal(np.argmax(predictions, 1), np.argmax(data.test_labels, 1))) print(f"The accuracy was {accuracy}", flush=True) time_used = time.time() - start_time return accuracy, time_used
def get_data(dataset): if dataset == "mnist": data = MNIST() elif dataset == "cifar": data = CIFAR() elif dataset == "tinyImagenet": data = TinyImagenet() elif dataset == "caltechSilhouettes": data = CaltechSiluettes() elif dataset == "GTSRB": data = GTSRB() elif dataset == "sign-language": data = SignLanguage() elif dataset == "rockpaperscissors": data = RockPaperScissors() else: raise NameError(f"{dataset} is not a valid dataset") return data
def convert(file_name, new_name, cifar=False): if not cifar: eq_weights, new_params = get_weights(file_name) data = MNIST() else: eq_weights, new_params = get_weights(file_name, inp_shape=(32, 32, 3)) data = CIFAR() model = Sequential() model.add(Flatten(input_shape=data.train_data.shape[1:])) for param in new_params: model.add(Dense(param)) model.add(Lambda(lambda x: tf.nn.relu(x))) model.add(Dense(10)) for i in range(len(eq_weights)): try: print(eq_weights[i][0].shape) except: pass model.layers[i].set_weights(eq_weights[i]) sgd = SGD(lr=0.01, decay=1e-5, momentum=0.9, nesterov=True) model.compile(loss=fn, optimizer=sgd, metrics=['accuracy']) model.save(new_name) acc = model.evaluate(data.validation_data, data.validation_labels)[1] printlog("Converting CNN to MLP") nlayer = file_name.split('_')[-3][0] filters = file_name.split('_')[-2] kernel_size = file_name.split('_')[-1] printlog( "model name = {0}, numlayer = {1}, filters = {2}, kernel size = {3}". format(file_name, nlayer, filters, kernel_size)) printlog("Model accuracy: {:.3f}".format(acc)) printlog("-----------------------------------") return acc
args.numlayer) + "layer_relu" + suffix # if still not found, try models/mnist_3layer_relu_1024_best if not os.path.isfile(modelfile): modelfile = "models/" + args.model + "_" + str( args.numlayer) + "layer_relu_" + str( nhidden) + "_best" + suffix if not os.path.isfile(modelfile): modelfile = args.filename if not os.path.isfile(modelfile): raise (RuntimeError("cannot find model file")) config = tf.ConfigProto() config.gpu_options.allow_growth = True with tf.Session(config=config) as sess: if args.model == "mnist": data = MNIST() if args.cnnmodel: model = nl.CNNModel(modelfile) elif args.filename: model = nl.NLayerModel(args.layers, modelfile) else: model = nl.NLayerModel([nhidden] * (args.numlayer - 1), modelfile) elif args.model == "cifar": data = CIFAR() if args.cnnmodel: model = nl.CNNModel(modelfile) elif args.filename: model = nl.NLayerModel(args.layers, modelfile, image_size=32,
def run(file_name, n_samples, p_n, q_n, activation='relu', cifar=False, tinyimagenet=False): np.random.seed(1215) tf.set_random_seed(1215) random.seed(1215) keras_model = load_model(file_name, custom_objects={ 'fn': fn, 'tf': tf, 'atan': tf.math.atan }) if tinyimagenet: model = CNNModel(keras_model, inp_shape=(64, 64, 3)) elif cifar: model = CNNModel(keras_model, inp_shape=(32, 32, 3)) else: model = CNNModel(keras_model) #Set correct linear_bounds function global linear_bounds if activation == 'relu': linear_bounds = relu_linear_bounds elif activation == 'ada': linear_bounds = ada_linear_bounds elif activation == 'sigmoid': linear_bounds = sigmoid_linear_bounds elif activation == 'tanh': linear_bounds = tanh_linear_bounds elif activation == 'arctan': linear_bounds = atan_linear_bounds start_time = time.time() upper_bound_conv.recompile() lower_bound_conv.recompile() compute_bounds.recompile() total_time = time.time() - start_time print("spent {} sec compiling bounds".format(total_time)) if cifar: inputs, targets, true_labels, true_ids, img_info = generate_data( CIFAR(), samples=n_samples, targeted=True, random_and_least_likely=True, target_type=0b0001, predictor=model.model.predict, start=0) elif tinyimagenet: inputs, targets, true_labels, true_ids, img_info = generate_data( tinyImagenet(), samples=n_samples, targeted=True, random_and_least_likely=True, target_type=0b0001, predictor=model.model.predict, start=0) else: inputs, targets, true_labels, true_ids, img_info = generate_data( MNIST(), samples=n_samples, targeted=True, random_and_least_likely=True, target_type=0b0001, predictor=model.model.predict, start=0) #0b01111 <- all #0b0010 <- random #0b0001 <- top2 #0b0100 <- least steps = 15 eps_0 = 0.05 summation = 0 start_time = time.time() warmup(model, inputs[0].astype(np.float32), eps_0, p_n, find_output_bounds) total_time = time.time() - start_time print("spent {} sec warming up".format(total_time)) start_time = time.time() for i in range(len(inputs)): #print('--- CNN-Cert: Computing eps for input image ' + str(i)+ '---') predict_label = np.argmax(true_labels[i]) target_label = np.argmax(targets[i]) weights = model.weights[:-1] biases = model.biases[:-1] shapes = model.shapes[:-1] W, b, s = model.weights[-1], model.biases[-1], model.shapes[-1] last_weight = (W[predict_label, :, :, :] - W[target_label, :, :, :]).reshape([1] + list(W.shape[1:])) weights.append(last_weight) biases.append(np.asarray([b[predict_label] - b[target_label]])) shapes.append((1, 1, 1)) #Perform binary search log_eps = np.log(eps_0) log_eps_min = -np.inf log_eps_max = np.inf for j in range(steps): LB, UB = find_output_bounds(weights, biases, shapes, model.pads, model.strides, inputs[i].astype(np.float32), np.exp(log_eps), p_n) #print("Step {}, eps = {:.5f}, {:.6s} <= f_c - f_t <= {:.6s}".format(j,np.exp(log_eps),str(np.squeeze(LB)),str(np.squeeze(UB)))) if LB > 0: #Increase eps log_eps_min = log_eps log_eps = np.minimum(log_eps + 1, (log_eps_max + log_eps_min) / 2) else: #Decrease eps log_eps_max = log_eps log_eps = np.maximum(log_eps - 1, (log_eps_max + log_eps_min) / 2) if p_n == 105: str_p_n = 'i' else: str_p_n = str(p_n) #print("[L1] method = CNN-Cert-{}, model = {}, image no = {}, true_id = {}, target_label = {}, true_label = {}, norm = {}, robustness = {:.5f}".format(activation,file_name, i, true_ids[i],target_label,predict_label,str_p_n,np.exp(log_eps_min))) summation += np.exp(log_eps_min) K.clear_session() eps_avg = summation / len(inputs) total_time = (time.time() - start_time) / len(inputs) print( "[L0] method = CNN-Cert-{}, model = {}, total images = {}, norm = {}, avg robustness = {:.5f}, avg runtime = {:.2f}" .format(activation, file_name, len(inputs), str_p_n, eps_avg, total_time)) return eps_avg, total_time
shuffle=True) # save model to a file if file_name != None: model.save(file_name) return {'model':model, 'history':history} if __name__ == '__main__': if not os.path.isdir('models'): os.makedirs('models') train(MNIST(), file_name="models/mnist_resnet_2", nlayer=2, activation = tf.nn.relu) train(MNIST(), file_name="models/mnist_resnet_3", nlayer=3, activation = tf.nn.relu) train(MNIST(), file_name="models/mnist_resnet_4", nlayer=4, activation = tf.nn.relu) train(MNIST(), file_name="models/mnist_resnet_5", nlayer=5, activation = tf.nn.relu) train(MNIST(), file_name="models/mnist_resnet_2_sigmoid", nlayer=2, activation = tf.sigmoid) train(MNIST(), file_name="models/mnist_resnet_3_sigmoid", nlayer=3, activation = tf.sigmoid) train(MNIST(), file_name="models/mnist_resnet_4_sigmoid", nlayer=4, activation = tf.sigmoid) train(MNIST(), file_name="models/mnist_resnet_5_sigmoid", nlayer=5, activation = tf.sigmoid) train(MNIST(), file_name="models/mnist_resnet_2_tanh", nlayer=2, activation = tf.tanh) train(MNIST(), file_name="models/mnist_resnet_3_tanh", nlayer=3, activation = tf.tanh) train(MNIST(), file_name="models/mnist_resnet_4_tanh", nlayer=4, activation = tf.tanh) train(MNIST(), file_name="models/mnist_resnet_5_tanh", nlayer=5, activation = tf.tanh) train(MNIST(), file_name="models/mnist_resnet_2_atan", nlayer=2, activation = tf.atan)
return tf.nn.softmax_cross_entropy_with_logits(labels=correct, logits=predicted / train_temp) sgd = SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True) model.compile(loss=fn, optimizer=sgd, metrics=['accuracy']) model.fit(data.train_data, data.train_labels, batch_size=batch_size, validation_data=(data.validation_data, data.validation_labels), epochs=num_epochs, shuffle=True) if file_name != None: model.save(file_name) return model if not os.path.isdir('models'): os.makedirs('models') if __name__ == '__main__': train(MNIST(), "models/mnist_cnn_lenet_nopool", [6, 16, 100], num_epochs=10, pool=False) train(MNIST(), "models/mnist_cnn_lenet", [6, 16, 100], num_epochs=10)