def mr(datasets, model, samples_path): """ :param datasets :param model :param samples_path :return: """ tf.reset_default_graph() X_train, Y_train, X_test, Y_test = get_data(datasets) input_shape, nb_classes = get_shape(datasets) sess, preds, x, y, model, feed_dict = model_load(datasets, model) preds_test = np.asarray([]) n_batches = int(np.ceil(1.0 * X_test.shape[0] / 256)) for i in range(n_batches): start = i * 256 end = np.minimum(len(X_test), (i + 1) * 256) preds_test = np.concatenate( (preds_test, model_argmax(sess, x, preds, X_test[start:end], feed=feed_dict))) inds_correct = np.asarray(np.where(preds_test != Y_test.argmax(axis=1))[0]) X_test = X_test[inds_correct] [image_list, image_files, real_labels, predicted_labels] = get_data_file(samples_path) for a in ua: if a in samples_path: result = len(image_files) / len(X_test) print('misclassification ratio is %.4f' % (result)) return result for a in ta: if a in samples_path: result = len(image_files) / (len(X_test) * (nb_classes - 1)) print('misclassification ratio is %.4f' % (result)) return result
def mutation_tutorial(datasets, attack, sample_path, store_path, model_path, level=1, test_num=100, mutation_number=1000, mutated=False): sess, preds, x, y, model, feed_dict = model_load(datasets, model_path + datasets) sample_path = sample_path + attack + '/' + datasets # sample_path = '../mt_result/mnist_jsma/adv_jsma' [image_list, image_files, real_labels, predicted_labels] = utils.get_data_mutation_test(sample_path) count = 0 for i in range(len(image_list)): ori_img = preprocess_image_1(image_list[i].astype('float64')) ori_img = np.expand_dims(ori_img.copy(), 0) p = model_argmax(sess, x, preds, ori_img, feed=feed_dict) if p != predicted_labels[i]: count = count + 1 image_file = image_files[i] os.remove("../datasets/adversary/" + attack + '/' + datasets + '/' + image_file) # os.remove(sample_path + '/' + image_file) # Close TF session print(count) sess.close() print('Finish.')
def detect(self, orig_img, orig_label, sess, x, preds, feed_dict=None): stop = False label_change_mutation_count = 0 total_mutation_count = 0 decided = False sprt_ratio = 0 while (not stop): total_mutation_count += 1 if total_mutation_count > self.max_mutation: # print('====== Result: Can\'t make a decision in ' + str(total_mutation_count-1) + ' mutations') # print('Total number of mutations evaluated: ', total_mutation_count-1) # print('Total label changes of the mutations: ', label_change_mutation_count) # print('=======') return False, decided, total_mutation_count, label_change_mutation_count # print('Test mutation number ', i) mt = MutationTest(self.img_rows, self.img_cols, 1, self.max_mutation, self.level) if self.rgb: mutation_matrix = mt.mutation_matrix(utils.generate_value_3) else: mutation_matrix = mt.mutation_matrix(utils.generate_value_1) mutation_img = orig_img.copy() + mutation_matrix * self.level mutation_img = np.expand_dims(mutation_img, 0) # mutation_img = orig_img + mutation_matrix * self.step_size total_mutation_count += 1 mu_label = model_argmax(sess, x, preds, mutation_img, feed=feed_dict) if orig_label != mu_label: # print('- Orig label: ', orig_label, ', New label: ', mu_label) label_change_mutation_count += 1 sprt_ratio = sprt_ratio + self.calculate_sprt_ratio( label_change_mutation_count, total_mutation_count) if sprt_ratio >= np.log((1 - self.beta) / self.alpha): # print('=== Result: Adversarial input ===') # print('Total number of mutations evaluated: ', total_mutation_count) # print('Total label changes of the mutations: ', label_change_mutation_count) # print('======') decided = True return True, decided, total_mutation_count, label_change_mutation_count elif sprt_ratio <= np.log(self.beta / (1 - self.alpha)): # print('=== Result: Normal input ===') # print('Total number of mutations evaluated: ', total_mutation_count) # print('Total label changes of the mutations: ', label_change_mutation_count) # print('======') decided = True return False, decided, total_mutation_count, label_change_mutation_count
def correct(datasets, model_name, X_test, Y_test, de=False, attack='fgsm', epoch=49): tf.reset_default_graph() sess, preds, x, y, model, feed_dict = model_load(datasets=datasets, model_name=model_name, de=de, attack=attack, epoch=epoch) preds_test = np.asarray([]) n_batches = int(np.ceil(1.0 * X_test.shape[0] / 256)) for i in range(n_batches): start = i * 256 end = np.minimum(len(X_test), (i + 1) * 256) preds_test = np.concatenate( (preds_test, model_argmax(sess, x, preds, X_test[start:end], feed=feed_dict))) inds_correct = np.asarray(np.where(preds_test == Y_test.argmax(axis=1))[0]) sess.close() tf.reset_default_graph() return inds_correct
def ric(datasets, model, samples_path, quality, epoch=49): """ :param datasets :param model :param samples_path :return: """ # Object used to keep track of (and return) key accuracies sess, preds, x, y, model, feed_dict = model_load(datasets, model, epoch=epoch) [image_list, image_files, real_labels, predicted_labels] = get_data_file(samples_path) ori_path = samples_path.replace('test_data', 'ric_ori') if not os.path.exists(ori_path): os.makedirs(ori_path) ic_path = samples_path.replace('test_data', 'ric_ic') if not os.path.exists(ic_path): os.makedirs(ic_path) count = 0 for i in range(len(image_list)): #jj=np.asarray(image_list[i:i+1]) #print(jj.shape) if datasets == 'mnist': adv_img_deprocessed = deprocess_image_1( np.asarray(image_list[i:i + 1]))[0] elif datasets == 'cifar10': adv_img_deprocessed = deprocess_image_1( np.asarray(image_list[i:i + 1])) saved_adv_image_path = os.path.join( ori_path, image_files[i].replace("npy", "png")) imsave(saved_adv_image_path, adv_img_deprocessed) output_IC_path = os.path.join(ic_path, image_files[i].replace("npy", "jpg")) cmd = '../../guetzli/bin/Release/guetzli --quality {} {} {}'.format( quality, saved_adv_image_path, output_IC_path) assert os.system( cmd ) == 0, 'guetzli tool should be install before, https://github.com/google/guetzli' if datasets == 'cifar10': IC_image = Image.open(output_IC_path).convert('RGB') IC_image = np.asarray( [np.array(IC_image).astype('float32') / 255.0]) #IC_image=IC_image.reshape(32, 32, 3) elif datasets == 'mnist': IC_image = Image.open(output_IC_path).convert('L') IC_image = np.expand_dims(np.array(IC_image).astype('float32'), axis=0) / 255.0 IC_image = IC_image.reshape(-1, 28, 28, 1) if model_argmax(sess, x, preds, IC_image, feed=feed_dict) != int( real_labels[i]): count = count + 1 result = 1.0 * count / len(image_list) print('Robustness to image compression is %.4f' % (result)) # Close TF session sess.close() return result
def batch_attack(datasets, attack, model_path, store_path, nb_classes): if 'mnist' == datasets: train_start = 0 train_end = 60000 test_start = 0 test_end = 10000 # Get MNIST test data X_train, Y_train, X_test, Y_test = data_mnist(train_start=train_start, train_end=train_end, test_start=test_start, test_end=test_end) elif 'cifar10' == datasets: preprocess_image = preprocess_image_1 train_start = 0 train_end = 50000 test_start = 0 test_end = 10000 # Get CIFAR10 test data X_train, Y_train, fn_train, X_test, Y_test, fn_test = data_cifar10( train_start=train_start, train_end=train_end, test_start=test_start, test_end=test_end, preprocess=preprocess_image) elif 'svhn' == datasets: # choose the method of preprocess image preprocess_image = preprocess_image_1 train_start = 0 train_end = 73257 test_start = 0 test_end = 26032 # Get SVHN test data X_train, Y_train, X_test, Y_test = data_svhn( train_start=train_start, train_end=train_end, test_start=test_start, test_end=test_end, preprocess=preprocess_image) store_path = store_path + attack + '/' + datasets sample_path = '../datasets/integration/batch_attack/' + datasets + '/' sess, preds, x, y, model, feed_dict = model_load(datasets, model_path) if os.listdir(sample_path) == []: for i in range(len(X_test)): sample = X_test[i:i + 1] path = sample_path + str(i) + '.png' imsave(path, deprocess_image_1(sample)) current_img = ndimage.imread(path) img = np.expand_dims( preprocess_image_1(current_img.astype('float64')), 0) p = model_argmax(sess, x, preds, img, feed=feed_dict) if p != Y_test[i].argmax(axis=0): os.remove(path) # for i in range(len(X_test)): # sample = X_test[i:i+1] # if model_argmax(sess, x, preds, sample, feed=feed_dict) == Y_test[i].argmax(axis=0): # path = sample_path + str(i) + '.png' # imsave(path, deprocess_image_1(sample)) sess.close() samples = os.listdir(sample_path) for sample in samples: tf.reset_default_graph() if 'blackbox' == attack: blackbox(datasets=datasets, sample_path=sample_path + sample, model_path=model_path, store_path=store_path, nb_classes=nb_classes) elif 'fgsm' == attack: fgsm(datasets=datasets, sample_path=sample_path + sample, model_path=model_path, store_path=store_path, nb_classes=nb_classes) else: i = int(sample.split('.')[-2]) for j in range(nb_classes): tf.reset_default_graph() if Y_test[i][j] == 0: if 'jsma' == attack: jsma(datasets=datasets, sample_path=sample_path + sample, target=j, model_path=model_path, store_path=store_path, nb_classes=nb_classes) if 'cw' == attack: cw(datasets=datasets, sample_path=sample_path + sample, target=j, model_path=model_path, store_path=store_path, nb_classes=nb_classes)
def bim(datasets, sample, model_name, store_path, step_size='0.3', batch_size=256, epoch=9): """ :param datasets :param sample: inputs to attack :param target: the class want to generate :param nb_classes: number of output classes :return: """ tf.reset_default_graph() X_train, Y_train, X_test, Y_test = get_data(datasets) input_shape, nb_classes = get_shape(datasets) print(epoch) sess, preds, x, y, model, feed_dict = model_load(datasets, model_name, epoch=epoch) ########################################################################### # Craft adversarial examples using the BIM approach ########################################################################### # Initialize the Basic Iterative Method (BIM) attack object and # graph ''' if 'mnist' == datasets: #sample = np.asarray([np.asarray(imread(sample_path)).reshape(28,28,1)]).astype('float32') #sample = preprocess_image_1(sample) print('1') elif 'cifar10' == datasets: sample = np.asarray([np.asarray(imread(sample_path)).reshape(32,32,3)]).astype('float32') sample = preprocess_image_1(sample) elif 'svhn' == datasets: sample = np.asarray([np.asarray(imread(sample_path)).reshape(32,32,3)]).astype('float32') sample = preprocess_image_1(sample) #print(sample.shape) ''' probabilities = model_prediction(sess, x, preds, sample, feed=feed_dict) if sample.shape[0] == 1: current_class = np.argmax(probabilities) else: current_class = np.argmax(probabilities, axis=1) if not os.path.exists(store_path): os.makedirs(store_path) # only for correct: acc_pre_index = [] for i in range(0, sample.shape[0]): if current_class[i] == np.argmax(Y_test[i]): acc_pre_index.append(i) sample_acc = np.zeros(shape=(len(acc_pre_index), input_shape[1], input_shape[2], input_shape[3]), dtype='float32') probabilities_acc = np.zeros(shape=(len(acc_pre_index), nb_classes), dtype='float32') current_class_acc = np.zeros(shape=(len(acc_pre_index)), dtype=int) for i in range(0, len(acc_pre_index)): sample_acc[i] = sample[acc_pre_index[i]] probabilities_acc[i] = probabilities[acc_pre_index[i]] current_class_acc[i] = current_class[acc_pre_index[i]] print('Start generating adv. example') #print(float(step_size)) if 'mnist' == datasets: bim_params = { 'eps': float(step_size), 'eps_iter': float(step_size) / 6, 'clip_min': 0., 'clip_max': 1. } elif 'cifar10' == datasets: bim_params = { 'eps': float(step_size), 'eps_iter': float(step_size) / 6, 'clip_min': 0., 'clip_max': 1. } elif 'svhn' == datasets: bim_params = { 'eps': float(step_size), 'eps_iter': float(step_size) / 6, 'clip_min': 0., 'clip_max': 1. } bim = BasicIterativeMethod(model, sess=sess) adv_x = bim.generate(x, **bim_params) nb_batches = int(math.ceil(float(sample_acc.shape[0]) / batch_size)) suc = 0 for batch in range(nb_batches): #start, end = batch_indices(batch, sample_acc.shape[0], batch_size) print(batch) start = batch * batch_size end = (batch + 1) * batch_size if end > sample_acc.shape[0]: end = sample_acc.shape[0] adv = sess.run(adv_x, feed_dict={ x: sample_acc[start:end], y: probabilities_acc[start:end] }) #adv_img_deprocessed = deprocess_image_1(adv) #adv:float 0-1 numpy.save("filename.npy",a) # Check if success was achieved #probabilities = model_prediction(sess, x, preds, sample, feed=feed_dict) new_class_label = model_argmax( sess, x, preds, adv, feed=feed_dict) # Predicted class of the generated adversary for i in range(0, len(new_class_label)): j = batch * batch_size + i if new_class_label[i] != current_class_acc[j]: suc += 1 path = store_path + '/' + str(acc_pre_index[j]) + '_' + str( time.time() * 1000) + '_' + str( current_class_acc[j]) + '_' + str(new_class_label[i]) np.save(path, adv[i]) # adv_img_deprocessed = deprocess_image_1(adv[i:i+1]) # adv_img_deprocessed=adv_img_deprocessed.reshape(adv_img_deprocessed.shape[1],adv_img_deprocessed.shape[2]) # path = store_path + '/' + str(acc_pre_index[j]) + '_' + str(time.time()*1000) + '_' + str(current_class_acc[j]) + '_' + str(new_class_label[i])+'.png' #print(adv[i].shape) # imsave(path, adv_img_deprocessed) # Close TF session sess.close() return suc, len(acc_pre_index)
def jsma(datasets, sample, model_name, target, store_path, gamma=0.1, start=0, end=10000, batch_size=32, epoch=9, mu=False, mu_var='gf', de=False, attack='fgsm'): """ the Jacobian-based saliency map approach (JSMA) :param datasets :param sample: inputs to attack :param target: the class want to generate :param nb_classes: number of output classes :return: """ tf.reset_default_graph() X_train, Y_train, X_test, Y_test = get_data(datasets) # sess, preds, x, y, model, feed_dict = model_load(datasets, model_name, epoch=epoch) sess, preds, x, y, model, feed_dict = model_load(datasets, model_name, epoch=epoch, mu=mu, mu_var=mu_var, de=de, attack=attack) ########################################################################### # Craft adversarial examples using the Jacobian-based saliency map approach ########################################################################### ''' if 'mnist' == datasets: sample = np.asarray([np.asarray(imread(sample_path)).reshape(28,28,1)]).astype('float32') sample = preprocess_image_1(sample) elif 'cifar10' == datasets: sample = np.asarray([np.asarray(imread(sample_path)).reshape(32,32,3)]).astype('float32') sample = preprocess_image_1(sample) elif 'svhn' == datasets: sample = np.asarray([np.asarray(imread(sample_path)).reshape(32,32,3)]).astype('float32') sample = preprocess_image_1(sample) ''' input_shape, nb_classes = get_shape(datasets) sample = sample[start:end] probabilities = model_prediction(sess, x, preds, sample, feed=feed_dict) current_class = [] for i in range(0, probabilities.shape[0]): current_class.append(np.argmax(probabilities[i])) if not os.path.exists(store_path): os.makedirs(store_path) ''' if target == current_class: return 'The target is equal to its original class' elif target >= nb_classes or target < 0: return 'The target is out of range' ''' #only for correct: Y_test = Y_test[start:end] acc_pre_index = [] for i in range(0, sample.shape[0]): if current_class[i] == np.argmax(Y_test[i]): acc_pre_index.append(i) print('Start generating adv. example for target class %i' % target) sample_acc = np.zeros(shape=(len(acc_pre_index), input_shape[1], input_shape[2], input_shape[3]), dtype='float') current_class_acc = np.zeros(shape=(len(acc_pre_index)), dtype=int) for i in range(0, len(acc_pre_index)): sample_acc[i] = sample[acc_pre_index[i]] current_class_acc[i] = current_class[acc_pre_index[i]] #print('current_class_acc',current_class_acc) # Instantiate a SaliencyMapMethod attack object jsma = SaliencyMapMethod(model, back='tf', sess=sess) jsma_params = { 'theta': 1., 'gamma': gamma, 'clip_min': 0., 'clip_max': 1., 'y_target': None } # This call runs the Jacobian-based saliency map approach one_hot_target = np.zeros((1, nb_classes), dtype=np.float32) one_hot_target[0, target] = 1 jsma_params['y_target'] = one_hot_target suc = 0 nb_batches = int(math.ceil(float(sample_acc.shape[0]) / batch_size)) for batch in range(nb_batches): #print(batch) start_batch = batch * batch_size end_batch = (batch + 1) * batch_size if end_batch > sample_acc.shape[0]: end_batch = sample_acc.shape[0] adv_inputs = sample_acc[start_batch:end_batch] for j in range(start_batch, end_batch): if current_class_acc[j] != target: adv_input = adv_inputs[j - start_batch].reshape( 1, input_shape[1], input_shape[2], input_shape[3]) adv = jsma.generate_np(adv_input, **jsma_params) new_class_labels = model_argmax(sess, x, preds, adv, feed=feed_dict) res = int(new_class_labels == target) if res == 1: adv = adv.reshape(adv.shape[1], adv.shape[2], adv.shape[3]) #adv_img_deprocessed = deprocess_image_1(adv) #adv_img_deprocessed=adv_img_deprocessed.reshape(adv_img_deprocessed.shape[1],adv_img_deprocessed.shape[2]) suc += 1 path = store_path + '/' + str( start + acc_pre_index[j] ) + '_' + str(time.time() * 1000) + '_' + str( current_class_acc[j]) + '_' + str(new_class_labels) #path=store_path + '/' + str(j)+ '_'+ str(current_class_acc[j]) +'.png' #imsave(path, adv_img_deprocessed) np.save(path, adv) #print(adv.shape) # Close TF session sess.close() return suc, len(acc_pre_index)
def sc(datasets, model_name, samples_path, layer=-3, num_section=1000, de='False', attack='fgsm', epoch=9): X_train, Y_train, X_test, Y_test = get_data(datasets) if de == True: adv_train_path = '../adv_result/' + datasets + '/' + attack + '/' + model_name + '/train_data' [image_list_train, _, real_labels, _] = get_data_file(adv_train_path) samples_train = np.asarray(image_list_train) if datasets == "mnist": indexs = random.sample(range(len(samples_train)), int(len(samples_train)/12)) elif datasets == "cifar10": indexs = random.sample(range(len(samples_train)), int(len(samples_train) / 10)) train_samples = np.concatenate((samples_train[indexs], X_train[:5000])) store_path = "../suprise/" + datasets + "/" + model_name + "/" + attack + '/' else: train_samples = X_train[:5000] store_path = "../suprise/" + datasets + "/" + model_name + "/ori/" if not os.path.exists(store_path): os.makedirs(store_path) a_n_train = [] train_labels = [] n_batches = int(np.ceil(1.0 * train_samples.shape[0] / 512)) for num in range(n_batches): print(num) start = num * 512 end = np.minimum(len(train_samples), (num + 1) * 512) tf.reset_default_graph() sess, preds, x, y, model, feed_dict = model_load(datasets=datasets, model_name=model_name, de=de, attack=attack, epoch=epoch) train_labels = train_labels+model_argmax(sess, x, preds, train_samples[start:end], feed_dict).tolist() a_n_train = a_n_train+at_training(sess, x, train_samples[start:end], model, feed_dict, layer).tolist() sess.close() del sess, preds, x, y, model, feed_dict gc.collect() a_n_train = np.asarray(a_n_train) train_labels = np.asarray(train_labels) np.save(store_path + "a_n_train.npy", a_n_train) np.save(store_path + "train_labels.npy", train_labels) else: a_n_train = np.load(store_path + "a_n_train.npy") train_labels = np.load(store_path + "train_labels.npy") class_inds = {} for i in range(10): class_inds[i] = np.where(train_labels == i)[0] kdes_store_path=store_path+'kdes.npy' if os.path.exists(kdes_store_path): kdes = np.load(kdes_store_path).item() else: kdes = {} for i in range(10): scott_bw = pow(len(a_n_train[class_inds[i]]), -1.0/(len(a_n_train[0])+4)) kdes[i] = KernelDensity(kernel='gaussian', bandwidth=scott_bw).fit(a_n_train[class_inds[i]]) np.save(kdes_store_path, kdes) lsa = [] dsa = [] c = set(range(len(train_labels))) lsa_test_store_path=store_path+"lsa_test.npy" dsa_test_store_path=store_path+"dsa_test.npy" if os.path.exists(lsa_test_store_path) and os.path.exists(dsa_test_store_path): lsa=np.load(lsa_test_store_path).tolist() dsa=np.load(dsa_test_store_path).tolist() else: # X_test=X_test n_batches = int(np.ceil(1.0 * X_test.shape[0] / 512)) for num in range(n_batches): print(num) start = num * 512 end = np.minimum(len(X_test), (num + 1) * 512) batch_samples = X_test[start:end] tf.reset_default_graph() sess, preds, x, y, model, feed_dict = model_load(datasets=datasets, model_name=model_name, de=de, attack=attack, epoch=epoch) batch_labels = model_argmax(sess, x, preds, batch_samples, feed=feed_dict) a_n_test = at_training(sess, x, batch_samples, model, feed_dict, layer) sess.close() del sess, preds, x, y, model, feed_dict gc.collect() for i in range(len(batch_samples)): kd_value = kdes[batch_labels[i]].score_samples(np.reshape(a_n_test[i],(1,-1)))[0] #/ len(a_n_train[class_inds[batch_labels[i]]]) lsa.append(-kd_value) data = np.asarray([a_n_test[i]], dtype=np.float32) batch = np.asarray(a_n_train[class_inds[batch_labels[i]]], dtype=np.float32) # dist = np.linalg.norm(data - batch, axis=1) dist = cdist(data, batch)[0] dist_a = min(dist) alpha_a = np.asarray([batch[np.argmin(dist)]]) c_i = set(class_inds[batch_labels[i]]) c_ni = list(c^c_i) batch = np.asarray(a_n_train[c_ni], dtype=np.float32) # dist_b = min(np.linalg.norm(alpha_a - batch, axis=1)) dist_b = min(cdist(alpha_a, batch)[0]) dsa.append(dist_a / dist_b) np.save(store_path + "lsa_test.npy", np.asarray(lsa)) np.save(store_path + "dsa_test.npy", np.asarray(dsa)) upper_lsa_test=max(lsa) lower_lsa_test = min(lsa) upper_dsa_test=max(dsa) lower_dsa_test = min(dsa) if samples_path not in ['test']: lsa = [] dsa = [] [image_list, _, _, _] = get_data_file(samples_path) # image_files, real_labels, predicted_labels samples_adv = np.asarray(image_list) n_batches = int(np.ceil(1.0 * samples_adv.shape[0] / 512)) for num in range(n_batches): print(num) start = num * 512 end = np.minimum(len(samples_adv), (num + 1) * 512) batch_samples = samples_adv[start:end] tf.reset_default_graph() sess, preds, x, y, model, feed_dict = model_load(datasets=datasets, model_name=model_name, de=de, attack=attack, epoch=epoch) batch_labels = model_argmax(sess, x, preds, batch_samples, feed=feed_dict) a_n_adv = at_training(sess, x, batch_samples, model, feed_dict, layer) sess.close() del sess, preds, x, y, model, feed_dict gc.collect() for i in range(len(batch_samples)): kd_value = kdes[batch_labels[i]].score_samples(np.reshape(a_n_adv[i],(1,-1)))[0] #/ len(a_n_train[class_inds[batch_labels[i]]]) lsa.append(-kd_value) data = np.asarray([a_n_adv[i]], dtype=np.float32) batch = np.asarray(a_n_train[class_inds[batch_labels[i]]], dtype=np.float32) # dist = np.linalg.norm(data - batch, axis=1) dist = cdist(data, batch)[0] dist_a = min(dist) alpha_a = np.asarray([batch[np.argmin(dist)]]) c_i = set(class_inds[batch_labels[i]]) c_ni = list(c ^ c_i) batch = np.asarray(a_n_train[c_ni], dtype=np.float32) # dist_b = min(np.linalg.norm(alpha_a - batch, axis=1)) dist_b = min(cdist(alpha_a, batch)[0]) dsa.append(dist_a / dist_b) for i in range(len(lsa)): lsa[i]=(lsa[i]-lower_lsa_test)/(upper_lsa_test-lower_lsa_test) dsa[i]=(dsa[i]-lower_dsa_test)/(upper_dsa_test-lower_dsa_test) lsa_mean = np.mean(lsa) lsa_std = np.std(lsa) dsa_mean = np.mean(dsa) dsa_std = np.std(dsa) half_section = int(num_section / 2) n_section_lsa=np.zeros(num_section).astype('int64') n_section_dsa=np.zeros(num_section).astype('int64') for i in range(len(lsa)): l = lsa[i]*half_section d = dsa[i]*half_section if math.ceil(l) < num_section and math.floor(l) >= 0: if math.ceil(l) == math.floor(l): n_section_lsa[int(l)-1]=1 else: n_section_lsa[int(l)]=1 if math.ceil(d) < num_section and math.floor(d) >= 0: if math.ceil(d) == math.floor(d): n_section_dsa[int(d)-1]=1 else: n_section_dsa[int(d)]=1 cov_lsa_1=1.0 * sum(n_section_lsa[:half_section])/(half_section) cov_dsa_1=1.0 * sum(n_section_dsa[:half_section])/(half_section) cov_lsa_2 = 1.0 * sum(n_section_lsa[half_section:]) / (half_section) cov_dsa_2 = 1.0 * sum(n_section_dsa[half_section:]) / (half_section) print([lsa_mean, lsa_std, cov_lsa_1, cov_lsa_2, upper_lsa_test,lower_lsa_test, dsa_mean, dsa_std, cov_dsa_1, cov_dsa_2, upper_dsa_test,lower_dsa_test]) return [lsa_mean, lsa_std, cov_lsa_1, cov_lsa_2, upper_lsa_test,lower_lsa_test, dsa_mean, dsa_std, cov_dsa_1, cov_dsa_2, upper_dsa_test,lower_dsa_test]
def mutation_test_ori(self, result, image_list, sess, x, preds, feed_dict=None): store_string = '' label_change_numbers = [] # Iterate over all the test data for i in range(len(image_list)): ori_img = image_list[i] orig_label = model_argmax(sess, x, preds, np.asarray([image_list[i]])) # pxzhang feed = {x: np.expand_dims(ori_img.copy(), 0)} if feed_dict is not None: feed.update(feed_dict) probabilities = sess.run(preds, feed)[0] max_p = max(probabilities) # new = utils.input_preprocessing(preds, x, 0.001, 0.0, 1.0) # ori_img = sess.run(new, feed_dict={x: np.expand_dims(ori_img.copy(), 0)})[0] label_changes = 0 imgs = np.asarray([ori_img.tolist()] * self.mutation_number) mu_imgs = imgs + np.asarray(self.mutations) * self.level n_batches = int(np.ceil(1.0 * mu_imgs.shape[0] / 256)) mu_labels = [] for j in range(n_batches): start = j * 256 end = np.minimum(len(mu_imgs), (j + 1) * 256) mu_labels = mu_labels + model_argmax( sess, x, preds, mu_imgs[start:end], feed=feed_dict).tolist() for mu_label in mu_labels: if mu_label != int(orig_label): label_changes += 1 # for j in range(self.mutation_number): # img = ori_img.copy() # add_mutation = self.mutations[j][0] # mu_img = img + add_mutation # # # Predict the label for the mutation # mu_img = np.expand_dims(mu_img, 0) # # mu_label = model_argmax(sess, x, preds, mu_img, feed=feed_dict) # # if mu_label != int(orig_label): # label_changes += 1 label_change_numbers.append(label_changes) # pxzhang store_string = store_string + str(i) + "," + str( orig_label) + "," + str(max_p) + "," + str( label_changes) + "\n" label_change_numbers = np.asarray(label_change_numbers) adv_average = round(np.mean(label_change_numbers), 2) adv_std = np.std(label_change_numbers) adv_99ci = round( 2.576 * adv_std / math.sqrt(len(label_change_numbers)), 2) result = result + 'ori,' + str(adv_average) + ',' + str( round(adv_std, 2)) + ',' + str(adv_99ci) + '\n' return store_string, result
# Craft adversarial examples using Carlini and Wagner's approach ########################################################################### ''' if 'mnist' == datasets: sample = np.asarray([np.asarray(imread(sample_path)).reshape(28, 28, 1)]).astype('float32') sample = preprocess_image_1(sample) elif 'cifar10' == datasets: sample = np.asarray([np.asarray(imread(sample_path)).reshape(32, 32, 3)]).astype('float32') sample = preprocess_image_1(sample) elif 'svhn' == datasets: sample = np.asarray([np.asarray(imread(sample_path)).reshape(32,32,3)]).astype('float32') sample = preprocess_image_1(sample) ''' input_shape, nb_classes = get_shape(datasets) current_class = model_argmax(sess, x, preds, sample, feed=feed_dict) #if not os.path.exists(store_path): # os.makedirs(store_path) if target == current_class: return 'The target is equal to its original class' elif target >= nb_classes or target < 0: return 'The target is out of range' print('succ@@@@@@@@@@@@@@@@@@@@@@@@@@@') print('Start generating adv. example for target class %i' % target) # Instantiate a CW attack object cw = CarliniWagnerL2(model, back='tf', sess=sess) one_hot = np.zeros((1, nb_classes), dtype=np.float32) one_hot[0, target] = 1
def blackbox(datasets, sample, model_name, submodel_name, store_path, step_size=0.3, batch_size=256): """ the black-box attack from arxiv.org/abs/1602.02697 :param datasets :param sample: inputs to attack :param target: the class want to generate :param nb_classes: number of output classes :return: """ # Simulate the black-box model locally # You could replace this by a remote labeling API for instance print("Preparing the black-box model.") tf.reset_default_graph() X_train, Y_train, X_test, Y_test = get_data(datasets) input_shape, nb_classes = get_shape(datasets) sess, bbox_preds, x, y, model, feed_dict = model_load(datasets, model_name) # Train substitute using method from https://arxiv.org/abs/1602.02697 print("Preparing the substitute model.") model_sub, preds_sub = sub_model_load(sess, datasets, submodel_name, model_name) ########################################################################### # Craft adversarial examples using the Blackbox approach ########################################################################### # Initialize the Fast Gradient Sign Method (FGSM) attack object. ''' if 'mnist' == datasets: sample = np.asarray([np.asarray(imread(sample_path)).reshape(28,28,1)]).astype('float32') sample = preprocess_image_1(sample) elif 'cifar10' == datasets: sample = np.asarray([np.asarray(imread(sample_path)).reshape(32,32,3)]).astype('float32') sample = preprocess_image_1(sample) elif 'svhn' == datasets: sample = np.asarray([np.asarray(imread(sample_path)).reshape(32,32,3)]).astype('float32') sample = preprocess_image_1(sample) ''' probabilities = model_prediction(sess, x, model(x), sample, feed=feed_dict) if sample.shape[0] == 1: current_class = np.argmax(probabilities) else: current_class = np.argmax(probabilities, axis=1) if not os.path.exists(store_path): os.makedirs(store_path) # only for correct: acc_pre_index = [] for i in range(0, sample.shape[0]): if current_class[i] == np.argmax(Y_test[i]): acc_pre_index.append(i) sample_acc = np.zeros(shape=(len(acc_pre_index), input_shape[1], input_shape[2], input_shape[3]), dtype='float32') probabilities_acc = np.zeros(shape=(len(acc_pre_index), nb_classes), dtype='float32') current_class_acc = np.zeros(shape=(len(acc_pre_index)), dtype=int) for i in range(0, len(acc_pre_index)): sample_acc[i] = sample[acc_pre_index[i]] probabilities_acc[i] = probabilities[acc_pre_index[i]] current_class_acc[i] = current_class[acc_pre_index[i]] if datasets == 'mnist': fgsm_par = { 'eps': step_size, 'ord': np.inf, 'clip_min': 0., 'clip_max': 1. } elif 'cifar10' == datasets: fgsm_par = { 'eps': step_size, 'ord': np.inf, 'clip_min': 0., 'clip_max': 1. } elif 'svhn' == datasets: fgsm_par = { 'eps': step_size, 'ord': np.inf, 'clip_min': 0., 'clip_max': 1. } fgsm = FastGradientMethod(model_sub, sess=sess) # Craft adversarial examples using the substitute x_adv_sub = fgsm.generate(x, **fgsm_par) nb_batches = int(math.ceil(float(sample_acc.shape[0]) / batch_size)) suc = 0 for batch in range(nb_batches): #start, end = batch_indices(batch, sample_acc.shape[0], batch_size) print(batch) start = batch * batch_size end = (batch + 1) * batch_size if end > sample_acc.shape[0]: end = sample_acc.shape[0] adv = sess.run(x_adv_sub, feed_dict={ x: sample_acc[start:end], y: probabilities_acc[start:end] }) adv_img_deprocessed = deprocess_image_1(adv) # Check if success was achieved #probabilities = model_prediction(sess, x, preds, sample, feed=feed_dict) new_class_label = model_argmax( sess, x, model(x), adv, feed=feed_dict) # Predicted class of the generated adversary for i in range(0, len(new_class_label)): j = batch * batch_size + i if new_class_label[i] != current_class_acc[j]: suc += 1 path = store_path + '/' + str(j) + '_' + str( time.time() * 1000) + '_' + str( current_class_acc[j]) + '_' + str( new_class_label[i]) + '.png' imsave(path, adv_img_deprocessed[i]) # Close TF session sess.close() return suc, len(acc_pre_index) '''
def cw(datasets, sample, model_name, target, store_path, ini_con=10, start=0, end=10000, batch_size=32, epoch=9, mu=False, mu_var='gf', de=False, attack='fgsm'): """ Carlini and Wagner's attack :param datasets :param sample: inputs to attack :param target: the class want to generate :param nb_classes: number of output classes :return: """ tf.reset_default_graph() X_train, Y_train, X_test, Y_test = get_data(datasets) # sess, preds, x, y, model, feed_dict = model_load(datasets, model_name, epoch=epoch) sess, preds, x, y, model, feed_dict = model_load(datasets, model_name, epoch=epoch, mu=mu, mu_var=mu_var, de=de, attack=attack) print('load successfule') ########################################################################### # Craft adversarial examples using Carlini and Wagner's approach ########################################################################### ''' if 'mnist' == datasets: sample = np.asarray([np.asarray(imread(sample_path)).reshape(28, 28, 1)]).astype('float32') sample = preprocess_image_1(sample) elif 'cifar10' == datasets: sample = np.asarray([np.asarray(imread(sample_path)).reshape(32, 32, 3)]).astype('float32') sample = preprocess_image_1(sample) elif 'svhn' == datasets: sample = np.asarray([np.asarray(imread(sample_path)).reshape(32,32,3)]).astype('float32') sample = preprocess_image_1(sample) ''' input_shape, nb_classes = get_shape(datasets) sample = sample[start:end] probabilities = model_prediction(sess, x, preds, sample, feed=feed_dict) current_class = [] for i in range(0, probabilities.shape[0]): current_class.append(np.argmax(probabilities[i])) if not os.path.exists(store_path): os.makedirs(store_path) ''' if target == current_class: return 'The target is equal to its original class' elif target >= nb_classes or target < 0: return 'The target is out of range' ''' #only for correct: Y_test = Y_test[start:end] #sample=sample[start:end] acc_pre_index = [] for i in range(0, sample.shape[0]): if current_class[i] == np.argmax(Y_test[i]): acc_pre_index.append(i) print('current_class', current_class) print('Start generating adv. example for target class %i' % target) # Instantiate a CW attack object sample_acc = np.zeros(shape=(len(acc_pre_index), input_shape[1], input_shape[2], input_shape[3]), dtype='float') current_class_acc = np.zeros(shape=(len(acc_pre_index)), dtype=int) for i in range(0, len(acc_pre_index)): sample_acc[i] = sample[acc_pre_index[i]] current_class_acc[i] = current_class[acc_pre_index[i]] print('current_class_acc', current_class_acc) cw = CarliniWagnerL2(model, back='tf', sess=sess) one_hot = np.zeros((1, nb_classes), dtype=np.float32) one_hot[0, target] = 1 adv_ys = one_hot yname = "y_target" if 'mnist' == datasets: cw_params = { 'binary_search_steps': 1, yname: adv_ys, 'max_iterations': 1000, 'learning_rate': 0.1, 'batch_size': 1, 'initial_const': ini_con } elif 'cifar10' == datasets: cw_params = { 'binary_search_steps': 1, yname: adv_ys, 'max_iterations': 1000, 'learning_rate': 0.1, 'batch_size': 1, 'initial_const': ini_con } suc = 0 nb_batches = int(math.ceil(float(sample_acc.shape[0]) / batch_size)) for batch in range(nb_batches): start_batch = batch * batch_size end_batch = (batch + 1) * batch_size if end_batch > sample_acc.shape[0]: end_batch = sample_acc.shape[0] adv_inputs = sample_acc[start_batch:end_batch] for j in range(start_batch, end_batch): if current_class_acc[j] != target: adv_input = adv_inputs[j - start_batch].reshape( 1, input_shape[1], input_shape[2], input_shape[3]) adv = cw.generate_np(adv_input, **cw_params) #print(adv.shape) #print(adv) new_class_labels = model_argmax(sess, x, preds, adv, feed=feed_dict) res = int(new_class_labels == target) if res == 1: adv = adv.reshape(adv.shape[1], adv.shape[2], adv.shape[3]) #adv_img_deprocessed = deprocess_image_1(adv) #adv_img_deprocessed=adv_img_deprocessed.reshape(adv_img_deprocessed.shape[1],adv_img_deprocessed.shape[2]) suc += 1 path = store_path + '/' + str( start + acc_pre_index[j] ) + '_' + str(time.time() * 1000) + '_' + str( current_class_acc[j]) + '_' + str(new_class_labels) #path = store_path + '/' + str(start+acc_pre_index[j]) + '_' + str(time.time()*1000) + '_' + str(current_class_acc[j]) + '_' + str(new_class_labels)+'.png' #path=store_path + '/' + str(j)+ '_'+ str(current_class_acc[j]) +'.png' #imsave(path, adv) np.save(path, adv) #print(adv.shape) sess.close() return suc, len(acc_pre_index)
def mutation_test_adv(self, preprocess_image, result, image_list, predicted_labels, sess, x, preds, image_files=None, feed_dict=None, mutate=False): store_string = '' mutated = False label_change_numbers = [] # Iterate over all the test data # count = 0 for i in range(len(image_list)): # if count % 100 == 0 and mutate: # path = '../mt_result/cifar10_test/mutation/' # if not os.path.exists(path): # os.makedirs(path) # path = path + image_files[i].split('_')[0] + '_' + str(count) # if not os.path.exists(path): # os.makedirs(path) # self.mutation_generate(mutated, path, utils.generate_value_3) # count += 1 ori_img = preprocess_image(image_list[i].astype('float64')) # ori_img = image_list[i] orig_label = predicted_labels[i] # pxzhang feed = {x: np.expand_dims(ori_img.copy(), 0)} if feed_dict is not None: feed.update(feed_dict) probabilities = sess.run(preds, feed)[0] max_p = max(probabilities) p = np.argmax(probabilities) # new = utils.input_preprocessing(preds, x, 0.001, 0.0, 1.0) # ori_img = sess.run(new, feed_dict={x: np.expand_dims(ori_img.copy(), 0)})[0] label_changes = 0 imgs = np.asarray([ori_img.tolist()] * self.mutation_number) mu_imgs = imgs + np.asarray(self.mutations) * self.level n_batches = int(np.ceil(1.0 * mu_imgs.shape[0] / 256)) mu_labels = [] for j in range(n_batches): start = j * 256 end = np.minimum(len(mu_imgs), (j + 1) * 256) mu_labels = mu_labels + model_argmax( sess, x, preds, mu_imgs[start:end], feed=feed_dict).tolist() for mu_label in mu_labels: if mu_label != int(orig_label): label_changes += 1 # for j in range(self.mutation_number): # img = ori_img.copy() # add_mutation = self.mutations[j]#[0] # mu_img = img + add_mutation # # # Predict the label for the mutation # mu_img = np.expand_dims(mu_img, 0) # # mu_label = model_argmax(sess, x, preds, mu_img, feed=feed_dict) # # if mu_label != int(orig_label): # label_changes += 1 label_change_numbers.append(label_changes) # pxzhang store_string = store_string + image_files[i] + "," + str( p) + "," + str(max_p) + "," + str(label_changes) + "\n" label_change_numbers = np.asarray(label_change_numbers) adv_average = round(np.mean(label_change_numbers), 2) adv_std = np.std(label_change_numbers) adv_99ci = round( 2.576 * adv_std / math.sqrt(len(label_change_numbers)), 2) if image_files == None: result = result + 'adv,' + ',' + str(adv_average) + ',' + str( round(adv_std, 2)) + ',' + str(adv_99ci) + '\n' else: result = result + 'adv_' + image_files[0].split( '_')[0] + ',' + str(adv_average) + ',' + str(round( adv_std, 2)) + ',' + str(adv_99ci) + '\n' return store_string, result
def detect_adv_samples(datasets, model_path, sample_path, store_path, attack_type): print('Loading the data and model...') # Load the model sess, preds, x, y, model, feed_dict = model_load(datasets, model_path) # # Load the dataset if 'mnist' == datasets: train_start = 0 train_end = 60000 test_start = 0 test_end = 10000 # Get MNIST test data X_train, Y_train, X_test, Y_test = data_mnist(train_start=train_start, train_end=train_end, test_start=test_start, test_end=test_end) elif 'cifar10' == datasets: preprocess_image = preprocess_image_1 train_start = 0 train_end = 50000 test_start = 0 test_end = 10000 # Get CIFAR10 test data X_train, Y_train, fn_train, X_test, Y_test, fn_test = data_cifar10( train_start=train_start, train_end=train_end, test_start=test_start, test_end=test_end, preprocess=preprocess_image) # # Refine the normal, noisy and adversarial sets to only include samples for # # which the original version was correctly classified by the model # preds_test = model_argmax(sess, x, preds, X_test, feed=feed_dict) # inds_correct = np.where(preds_test == Y_test.argmax(axis=1))[0] # X_test = X_test[inds_correct] # X_test = X_test[np.random.choice(len(X_test), 500)]#500 # # # Check attack type, select adversarial and noisy samples accordingly # print('Loading adversarial samples...') # # Load adversarial samplesx # [X_test_adv, adv_image_files, real_labels, predicted_labels] = utils.get_data_mutation_test(sample_path) # X_test_adv = preprocess_image_1(np.asarray(X_test_adv).astype('float32')) # if len(X_test_adv.shape) < 4: # X_test_adv = np.expand_dims(X_test_adv, axis=3) [X_test_adv_train, adv_image_files, real_labels, predicted_labels ] = utils.get_data_mutation_test("../datasets/experiment/" + datasets + "/" + attack_type + "/train") [X_test_adv_test, adv_image_files, real_labels, predicted_labels ] = utils.get_data_mutation_test("../datasets/experiment/" + datasets + "/" + attack_type + "/test") train_num = len(X_test_adv_train) test_num = len(X_test_adv_test) X_test_adv = preprocess_image_1( np.concatenate((np.asarray(X_test_adv_train), np.asarray(X_test_adv_test))).astype('float32')) if len(X_test_adv.shape) < 4: X_test_adv = np.expand_dims(X_test_adv, axis=3) [X_test_train, adv_image_files, real_labels, predicted_labels] = utils.get_data_normal_test("../datasets/experiment/" + datasets + "/normal/train") [X_test_test, adv_image_files, real_labels, predicted_labels] = utils.get_data_normal_test("../datasets/experiment/" + datasets + "/normal/test") X_test_train = np.asarray(X_test_train)[np.random.choice(len(X_test_train), train_num, replace=False)] X_test_test = np.asarray(X_test_test)[np.random.choice(len(X_test_test), test_num, replace=False)] X_test = preprocess_image_1( np.concatenate((np.asarray(X_test_train), np.asarray(X_test_test))).astype('float32')) if len(X_test.shape) < 4: X_test = np.expand_dims(X_test, axis=3) ## Get Bayesian uncertainty scores print('Getting Monte Carlo dropout variance predictions...') uncerts_normal = get_mc_predictions(sess, x, preds, X_test).var(axis=0).mean(axis=1) uncerts_adv = get_mc_predictions(sess, x, preds, X_test_adv).var(axis=0).mean(axis=1) ## Get KDE scores # Get deep feature representations print('Getting deep feature representations...') X_train_features = get_deep_representations(sess, x, X_train, model, feed_dict) X_test_normal_features = get_deep_representations(sess, x, X_test, model, feed_dict) X_test_adv_features = get_deep_representations(sess, x, X_test_adv, model, feed_dict) # Train one KDE per class print('Training KDEs...') class_inds = {} for i in range(Y_train.shape[1]): class_inds[i] = np.where(Y_train.argmax(axis=1) == i)[0] kdes = {} warnings.warn( "Using pre-set kernel bandwidths that were determined " "optimal for the specific CNN models of the paper. If you've " "changed your model, you'll need to re-optimize the " "bandwidth.") for i in range(Y_train.shape[1]): kdes[i] = KernelDensity(kernel='gaussian', bandwidth=BANDWIDTHS[datasets]) \ .fit(X_train_features[class_inds[i]]) # Get model predictions print('Computing model predictions...') preds_test_normal = model_argmax(sess, x, preds, X_test, feed=feed_dict) preds_test_adv = model_argmax(sess, x, preds, X_test_adv, feed=feed_dict) # Get density estimates print('computing densities...') densities_normal = score_samples(kdes, X_test_normal_features, preds_test_normal) densities_adv = score_samples(kdes, X_test_adv_features, preds_test_adv) uncerts_pos = uncerts_adv[:] uncerts_neg = uncerts_normal[:] characteristics, labels = merge_and_generate_labels( uncerts_pos, uncerts_neg) file_name = os.path.join('../detection/bu/', "%s_%s.npy" % (datasets, attack_type)) data = np.concatenate((characteristics, labels), axis=1) np.save(file_name, data) densities_pos = densities_adv[:] densities_neg = densities_normal[:] characteristics, labels = merge_and_generate_labels( densities_pos, densities_neg) file_name = os.path.join( '../detection/de/', "%s_%s_%.4f.npy" % (datasets, attack_type, BANDWIDTHS[datasets])) data = np.concatenate((characteristics, labels), axis=1) np.save(file_name, data) ## Z-score the uncertainty and density values uncerts_normal_z, uncerts_adv_z = normalize(uncerts_normal, uncerts_adv) densities_normal_z, densities_adv_z = normalize(densities_normal, densities_adv) ## Build detector values, labels = features(densities_pos=densities_adv_z, densities_neg=densities_normal_z, uncerts_pos=uncerts_adv_z, uncerts_neg=uncerts_normal_z) X_tr, Y_tr, X_te, Y_te = block_split(values, labels, train_num) lr = train_lr(X_tr, Y_tr) ## Evaluate detector # Compute logistic regression model predictions probs = lr.predict_proba(X_te)[:, 1] preds = lr.predict(X_te) # Compute AUC n_samples = int(len(X_te) / 2) # The first 2/3 of 'probs' is the negative class (normal and noisy samples), # and the last 1/3 is the positive class (adversarial samples). _, _, auc_score = compute_roc(probs_neg=probs[:n_samples], probs_pos=probs[n_samples:]) precision = precision_score(Y_te, preds) recall = recall_score(Y_te, preds) y_label_pred = lr.predict(X_te) acc = accuracy_score(Y_te, y_label_pred) print( 'Detector ROC-AUC score: %0.4f, accuracy: %.4f, precision: %.4f, recall: %.4f' % (auc_score, acc, precision, recall))
def prepare_datasets(datasets, model_path, attack_type, sample_path): print('Loading the data and model...') # Load the model sess, preds, x, y, model, feed_dict = model_load(datasets, model_path) # Load the dataset if 'mnist' == datasets: train_start = 0 train_end = 60000 test_start = 0 test_end = 10000 # Get MNIST test data X_train, Y_train, X_test, Y_test = data_mnist(train_start=train_start, train_end=train_end, test_start=test_start, test_end=test_end) elif 'cifar10' == datasets: preprocess_image = preprocess_image_1 train_start = 0 train_end = 50000 test_start = 0 test_end = 10000 # Get CIFAR10 test data X_train, Y_train, fn_train, X_test, Y_test, fn_test = data_cifar10( train_start=train_start, train_end=train_end, test_start=test_start, test_end=test_end, preprocess=preprocess_image) if attack_type == "normal": # Refine the normal, noisy and adversarial sets to only include samples for # which the original version was correctly classified by the model preds_test = np.asarray([]) for i in range(40): preds_test = np.concatenate( (preds_test, model_argmax(sess, x, preds, X_test[i * 250:(i + 1) * 250], feed=feed_dict))) inds_correct = np.asarray( np.where(preds_test == Y_test.argmax(axis=1))[0]) inds_correct = inds_correct[np.random.choice(len(inds_correct), 5000, replace=False)] X_test = X_test[inds_correct] for i in range(4000): imsave( "../datasets/experiment/" + datasets + "/normal/train/" + str(inds_correct[i]) + '_' + str(int(preds_test[inds_correct[i]])) + '_' + str(int(preds_test[inds_correct[i]])) + '_.png', deprocess_image_1(X_test[i:i + 1])) for j in range(1000): imsave( "../datasets/experiment/" + datasets + "/normal/test/" + str(inds_correct[4000 + j]) + '_' + str(int(preds_test[inds_correct[4000 + j]])) + '_' + str(int(preds_test[inds_correct[4000 + j]])) + '_.png', deprocess_image_1(X_test[4000 + j:4001 + j])) elif attack_type == "error": preds_test = np.asarray([]) for i in range(40): preds_test = np.concatenate( (preds_test, model_argmax(sess, x, preds, X_test[i * 250:(i + 1) * 250], feed=feed_dict))) inds_correct = np.asarray( np.where(preds_test != Y_test.argmax(axis=1))[0]) X_test = X_test[inds_correct] num = int(len(X_test) * 0.8) for i in range(num): imsave( "../datasets/experiment/" + datasets + "/error/train/" + str(inds_correct[i]) + '_' + str(int(np.argmax(Y_test[inds_correct[i]]))) + '_' + str(int(preds_test[inds_correct[i]])) + '_.png', deprocess_image_1(X_test[i:i + 1])) for j in range(len(X_test) - num): imsave( "../datasets/experiment/" + datasets + "/error/test/" + str(inds_correct[num + j]) + '_' + str(int(np.argmax(Y_test[inds_correct[num + j]]))) + '_' + str(int(preds_test[inds_correct[num + j]])) + '_.png', deprocess_image_1(X_test[num + j:num + 1 + j])) else: # Check attack type, select adversarial and noisy samples accordingly print('Loading adversarial samples...') # Load adversarial samplesx [X_test_adv, adv_image_files, real_labels, predicted_labels ] = utils.get_data_mutation_test(sample_path + attack_type + '/' + datasets) if len(X_test_adv) > 5000: index = np.asarray(range(len(X_test_adv))) index = index[np.random.choice(len(index), 5000, replace=False)] for i in range(4000): imsave( "../datasets/experiment/" + datasets + "/" + attack_type + "/train/" + adv_image_files[index[i]], X_test_adv[index[i]]) for j in range(1000): imsave( "../datasets/experiment/" + datasets + "/" + attack_type + "/test/" + adv_image_files[index[4000 + j]], X_test_adv[index[4000 + j]]) else: index = np.asarray(range(len(X_test_adv))) np.random.shuffle(index) cut = int(len(X_test_adv) * 0.8) for i in range(len(index)): if i < cut: imsave( "../datasets/experiment/" + datasets + "/" + attack_type + "/train/" + adv_image_files[index[i]], X_test_adv[index[i]]) else: imsave( "../datasets/experiment/" + datasets + "/" + attack_type + "/test/" + adv_image_files[index[i]], X_test_adv[index[i]])
def mutation_tutorial(datasets, attack, sample_path, store_path, model_path, level=1, test_num=100, mutation_number=1000, mutated=False): if 'mnist' == datasets: train_start = 0 train_end = 60000 test_start = 0 test_end = 10000 # Get MNIST test data X_train, Y_train, X_test, Y_test = data_mnist(train_start=train_start, train_end=train_end, test_start=test_start, test_end=test_end) elif 'cifar10' == datasets: preprocess_image = preprocess_image_1 train_start = 0 train_end = 50000 test_start = 0 test_end = 10000 # Get CIFAR10 test data X_train, Y_train, fn_train, X_test, Y_test, fn_test = data_cifar10( train_start=train_start, train_end=train_end, test_start=test_start, test_end=test_end, preprocess=preprocess_image) elif 'svhn' == datasets: # choose the method of preprocess image preprocess_image = preprocess_image_1 train_start = 0 train_end = 73257 test_start = 0 test_end = 26032 # Get SVHN test data X_train, Y_train, X_test, Y_test = data_svhn( train_start=train_start, train_end=train_end, test_start=test_start, test_end=test_end, preprocess=preprocess_image) sess, preds, x, y, model, feed_dict = model_load(datasets, model_path + datasets) # Generate random matution matrix for mutations store_path = store_path + attack + '/' + datasets + '/' + str(level) if not os.path.exists(store_path): os.makedirs(store_path) result = '' sample_path = sample_path + attack + '/' + datasets [image_list, image_files, real_labels, predicted_labels] = utils.get_data_mutation_test(sample_path) index = np.random.choice(len(image_files), test_num, replace=False) image_list = np.asarray(image_list)[index] image_files = np.asarray(image_files)[index].tolist() predicted_labels = np.asarray(predicted_labels)[index].tolist() seed_number = len(image_list) if datasets == 'mnist': img_rows = 28 img_cols = 28 mutation_test = MutationTest(img_rows, img_cols, seed_number, mutation_number, level) mutation_test.mutation_generate(mutated, store_path, utils.generate_value_1) elif datasets == 'cifar10' or datasets == 'svhn': img_rows = 32 img_cols = 32 mutation_test = MutationTest(img_rows, img_cols, seed_number, mutation_number, level) mutation_test.mutation_generate(mutated, store_path, utils.generate_value_3) store_string, result = mutation_test.mutation_test_adv( preprocess_image_1, result, image_list, predicted_labels, sess, x, preds, image_files, feed_dict) with open(store_path + "/adv_result.csv", "w") as f: f.write(store_string) path = store_path + '/ori_jsma' if not os.path.exists(path): os.makedirs(path) preds_test = np.asarray([]) for i in range(40): preds_test = np.concatenate( (preds_test, model_argmax(sess, x, preds, X_test[i * 250:(i + 1) * 250], feed=feed_dict))) inds_correct = np.asarray(np.where(preds_test == Y_test.argmax(axis=1))[0]) inds_correct = inds_correct[np.random.choice(len(inds_correct), test_num, replace=False)] image_list = X_test[inds_correct] real_labels = Y_test[inds_correct].argmax(axis=1) np.save(path + '/ori_x.npy', np.asarray(image_list)) np.save(path + '/ori_y.npy', np.asarray(real_labels)) image_list = np.load(path + '/ori_x.npy') real_labels = np.load(path + '/ori_y.npy') store_string, result = mutation_test.mutation_test_ori( result, image_list, sess, x, preds, feed_dict) with open(store_path + "/ori_result.csv", "w") as f: f.write(store_string) with open(store_path + "/result.csv", "w") as f: f.write(result) # Close TF session sess.close() print('Finish.')
def jsma(datasets, sample_path, model_name, target, store_path='../mt_result/integration/jsma/mnist'): """ the Jacobian-based saliency map approach (JSMA) :param datasets :param sample: inputs to attack :param target: the class want to generate :param nb_classes: number of output classes :return: """ sess, preds, x, y, model, feed_dict = model_load(datasets, model_name) ########################################################################### # Craft adversarial examples using the Jacobian-based saliency map approach ########################################################################### if 'mnist' == datasets: sample = np.asarray( [np.asarray(imread(sample_path)).reshape(28, 28, 1)]).astype('float32') sample = preprocess_image_1(sample) elif 'cifar10' == datasets: sample = np.asarray( [np.asarray(imread(sample_path)).reshape(32, 32, 3)]).astype('float32') sample = preprocess_image_1(sample) elif 'svhn' == datasets: sample = np.asarray( [np.asarray(imread(sample_path)).reshape(32, 32, 3)]).astype('float32') sample = preprocess_image_1(sample) input_shape, nb_classes = get_shape(datasets) current_class = model_argmax(sess, x, preds, sample, feed=feed_dict) if not os.path.exists(store_path): os.makedirs(store_path) if target == current_class: return 'The target is equal to its original class' elif target >= nb_classes or target < 0: return 'The target is out of range' print('Start generating adv. example for target class %i' % target) # Instantiate a SaliencyMapMethod attack object jsma = SaliencyMapMethod(model, back='tf', sess=sess) jsma_params = { 'theta': 1., 'gamma': 0.1, 'clip_min': 0., 'clip_max': 1., 'y_target': None } # This call runs the Jacobian-based saliency map approach one_hot_target = np.zeros((1, nb_classes), dtype=np.float32) one_hot_target[0, target] = 1 jsma_params['y_target'] = one_hot_target adv_x = jsma.generate_np(sample, **jsma_params) # Check if success was achieved new_class_label = model_argmax( sess, x, preds, adv_x, feed=feed_dict) # Predicted class of the generated adversary res = int(new_class_label == target) # Close TF session sess.close() if res == 1: adv_img_deprocessed = deprocess_image_1(adv_x) i = sample_path.split('/')[-1].split('.')[-2] path = store_path + '/adv_' + str( time.time() * 1000) + '_' + i + '_' + str( current_class) + '_' + str(new_class_label) + '_.png' imsave(path, adv_img_deprocessed) print('$$$adv_img{' + path + '}') print('$$$ori_img{' + sample_path + '}')