def mr(datasets, model_name, attack, va, epoch=49): """ :param datasets :param sample: inputs to attack :param target: the class want to generate :param nb_classes: number of output classes :return: """ tf.reset_default_graph() X_train, Y_train, X_test, Y_test = get_data(datasets) input_shape, nb_classes = get_shape(datasets) sample = X_test sess, preds, x, y, model, feed_dict = model_load(datasets, model_name, epoch=epoch) probabilities = model_prediction(sess, x, preds, sample, feed=feed_dict, datasets=datasets) if sample.shape[0] == 1: current_class = np.argmax(probabilities) else: current_class = np.argmax(probabilities, axis=1) # only for correct: acc_pre_index = [] for i in range(0, sample.shape[0]): if current_class[i] == np.argmax(Y_test[i]): acc_pre_index.append(i) print(len(acc_pre_index)) sess.close() total = 0 if attack == 'fgsm': samples_path = '../adv_result/' + datasets + '/' + attack + '/' + model_name + '/' + str( va) [image_list, image_files, real_labels, predicted_labels] = get_data_file(samples_path) num = len(image_list) return num / len(acc_pre_index) else: total = 0 for tar in range(0, nb_classes): samples_path = '../adv_result/' + datasets + '/' + attack + '/' + model_name + '/' + str( va) + '_' + str(tar) [image_list, image_files, real_labels, predicted_labels] = get_data_file(samples_path) total += len(image_list) return total / len(acc_pre_index)
def pure(datasets='mnist', attack='fgsm', model_name='lenet1'): tf.reset_default_graph() samples_path = '../adv_result/' + datasets + '/' + attack + '/' + model_name + '/pure' if not os.path.isdir(samples_path): os.makedirs(samples_path + '/train') os.makedirs(samples_path + '/test') samples_path_train = '../adv_result/' + datasets + '/' + attack + '/' + model_name + '/train_data' samples_path_test = '../adv_result/' + datasets + '/' + attack + '/' + model_name + '/test_data' sess, preds, x, y, model, feed_dict = model_load(datasets, model_name) [ image_list_train, image_files_train, real_labels_train, predicted_labels_train ] = get_data_file(samples_path_train) [ image_list_test, image_files_test, real_labels_test, predicted_labels_test ] = get_data_file(samples_path_test) #samples_train = np.asarray([preprocess_image_1(image.astype('float64')) for image in image_list_train]) #samples_test = np.asarray([preprocess_image_1(image.astype('float64')) for image in image_list_test]) samples_train = np.asarray(image_list_train) samples_test = np.asarray(image_list_test) probabilities_train = model_prediction(sess, x, preds, samples_train, feed=feed_dict) probabilities_test = model_prediction(sess, x, preds, samples_test, feed=feed_dict) for i in range(0, samples_train.shape[0]): if predicted_labels_train[i] == np.argmax(probabilities_train[i]): pure_train = samples_path + '/train/' + image_files_train[i] #imsave(pure_train, image_list_train[i]) np.save(pure_train, image_list_train[i]) for i in range(0, samples_test.shape[0]): if predicted_labels_test[i] == np.argmax(probabilities_test[i]): pure_test = samples_path + '/test/' + image_files_test[i] #imsave(pure_test, image_list_test[i]) np.save(pure_test, image_list_test[i])
def actc(datasets, model, samples_path, epoch=49): """ :param datasets :param model :param samples_path :return: """ # Object used to keep track of (and return) key accuracies tf.reset_default_graph() sess, preds, x, y, model, feed_dict = model_load(datasets, model, epoch=epoch) [image_list, image_files, real_labels, predicted_labels] = get_data_file(samples_path) #samples = np.asarray([preprocess_image_1(image.astype('float64')) for image in image_list]) samples=np.asarray(image_list) pbs = [] n_batches = int(np.ceil(samples.shape[0] / 256)) for i in range(n_batches): start = i * 256 end = np.minimum(len(samples), (i + 1) * 256) feed = {x: samples[start:end]} if feed_dict is not None: feed.update(feed_dict) probabilities = sess.run(preds, feed) for j in range(len(probabilities)): pbs.append(probabilities[j][real_labels[start+j]]) result = sum(pbs) / len(pbs) print('average confidence of true class %.4f' %(result)) # Close TF session sess.close() return result
def mr(datasets, model, samples_path): """ :param datasets :param model :param samples_path :return: """ tf.reset_default_graph() X_train, Y_train, X_test, Y_test = get_data(datasets) input_shape, nb_classes = get_shape(datasets) sess, preds, x, y, model, feed_dict = model_load(datasets, model) preds_test = np.asarray([]) n_batches = int(np.ceil(1.0 * X_test.shape[0] / 256)) for i in range(n_batches): start = i * 256 end = np.minimum(len(X_test), (i + 1) * 256) preds_test = np.concatenate( (preds_test, model_argmax(sess, x, preds, X_test[start:end], feed=feed_dict))) inds_correct = np.asarray(np.where(preds_test != Y_test.argmax(axis=1))[0]) X_test = X_test[inds_correct] [image_list, image_files, real_labels, predicted_labels] = get_data_file(samples_path) for a in ua: if a in samples_path: result = len(image_files) / len(X_test) print('misclassification ratio is %.4f' % (result)) return result for a in ta: if a in samples_path: result = len(image_files) / (len(X_test) * (nb_classes - 1)) print('misclassification ratio is %.4f' % (result)) return result
def ass(datasets, model, samples_path): """ :param datasets :param model :param samples_path :return: """ tf.reset_default_graph() X_train, Y_train, X_test, Y_test = get_data(datasets) X_test = [deprocess_image_1(np.asarray([image])) for image in X_test] [image_list, image_files, real_labels, predicted_labels] = get_data_file(samples_path) if datasets == 'cifar10': image_list = [(img * 255).reshape(img.shape[0], img.shape[1], img.shape[2]) for img in image_list] else: image_list = [(img * 255).reshape(img.shape[0], img.shape[1]) for img in image_list] result = 0 for i in range(len(image_list)): index = int(image_files[i].split('_')[-4]) result = result + ssim(np.asarray(image_list[i]), np.asarray(X_test[index])) result = result / len(image_list) print('average structural similarity is %.4f' % (result)) return result
def ald(datasets, model, samples_path, p, epoch=49): """ :param datasets :param model :param samples_path :param p :return: """ tf.reset_default_graph() X_train, Y_train, X_test, Y_test = get_data(datasets) X_test = [deprocess_image_1(np.asarray([image])) for image in X_test] [image_list, image_files, real_labels, predicted_labels] = get_data_file(samples_path) if datasets=='cifar10': image_list = [(img*255).reshape(img.shape[0], img.shape[1], img.shape[2]) for img in image_list] else: image_list = [(img*255).reshape(img.shape[0], img.shape[1]) for img in image_list] distortion = 0 for i in range(len(image_list)): index = int(image_files[i].split('_')[-4]) distortion = distortion + distortion_measure(np.asarray(image_list[i]), np.asarray(X_test[index]), p) result = distortion / len(image_list) print('average L-%s distortion is %.4f' % (p, result)) return result
def psd(datasets, model, samples_path, n): """ :param datasets :param model :param samples_path :param n :return: """ con = 5 X_train, Y_train, X_test, Y_test = get_data(datasets) X_test = [deprocess_image_1(np.asarray([image])) for image in X_test] [image_list, image_files, real_labels, predicted_labels] = get_data_file(samples_path) # if datasets=='cifar10': # image_list = [(img*255).reshape(img.shape[0], img.shape[1], img.shape[2]) for img in image_list] # else: # image_list = [(img*255).reshape(img.shape[0], img.shape[1]) for img in image_list] if datasets == 'cifar10': image_list = [ deprocess_image_1(np.asarray([img ])).reshape(img.shape[0], img.shape[1], img.shape[2]) for img in image_list ] else: image_list = [ deprocess_image_1(np.asarray([img ])).reshape(img.shape[0], img.shape[1]) for img in image_list ] result = 0.0 for i in range(len(image_list)): index = int(image_files[i].split('_')[-4]) adv = np.asarray(image_list[i]) ori = np.asarray(X_test[index]) if datasets == 'cifar10': ori = ori.reshape(ori.shape[0], ori.shape[1], ori.shape[2]) else: ori = ori.reshape(ori.shape[1], ori.shape[2]) #result = result + distance(adv, ori, (n - 1) / 2) result = result + distance(adv, ori, n, con) print('average perturbation sensitivity distance is %.4f' % (result / len(image_list))) #print('average perturbation sensitivity distance is %.4f' % (result/100)) return result / len(image_list)
def test_adv(datasets, model_name, samples_path, de=True, attack='fgsm', epoch=9): [image_list, _, real_labels, _] = get_data_file(samples_path) tf.reset_default_graph() sess, preds, x, y, model, feed_dict = model_load(datasets=datasets, model_name=model_name, de=de, attack=attack, epoch=epoch) def y_one_hot(label): y = np.zeros(10) y[label] = 1 return y eval_params = {'batch_size': 256} labels_adv = np.asarray([y_one_hot(int(label)) for label in real_labels]) accuracy = model_eval(sess,x,y,preds,np.asarray(image_list), labels_adv, feed_dict, eval_params) print(accuracy)
def nte(datasets, model, samples_path, epoch=49): """ :param datasets :param model :param samples_path :return: """ tf.reset_default_graph() # Object used to keep track of (and return) key accuracies sess, preds, x, y, model, feed_dict = model_load(datasets, model, epoch=epoch) [image_list, image_files, real_labels, predicted_labels] = get_data_file(samples_path) samples = np.asarray([preprocess_image_1(image.astype('float64')) for image in image_list]) samples = np.asarray(image_list) pbs = [] n_batches = int(np.ceil(samples.shape[0] / 256)) for i in range(n_batches): start = i * 256 end = np.minimum(len(samples), (i + 1) * 256) feed = {x: samples[start:end]} if feed_dict is not None: feed.update(feed_dict) probabilities = sess.run(preds, feed) #print(probabilities[1]) for j in range(len(probabilities)): pro_adv_max=probabilities[j][predicted_labels[start+j]] temp=np.delete(probabilities[j], predicted_labels[start+j], axis=0) pro_adv_top2=np.max(temp) pbs.append(pro_adv_max-pro_adv_top2) result = sum(pbs) / len(pbs) print('Noise Tolerance Estimation %.4f' %(result)) # Close TF session sess.close() return result
def choose_data(attack='fgsm', datasets='mnist', total_num=10000, model_name='lenet1'): tf.set_random_seed(1234) config = tf.ConfigProto() #config.gpu_options.per_process_gpu_memory_fraction = 0.7 config.gpu_options.allow_growth = True sess = tf.Session(config=config) X_train, Y_train, X_test, Y_test = get_data(datasets) train_path = '../adv_result/' + datasets + '/' + attack + '/' + model_name store_path_train = '../adv_result/' + datasets + '/' + attack + '/' + model_name + '/train_data' store_path_test = '../adv_result/' + datasets + '/' + attack + '/' + model_name + '/test_data' if not os.path.isdir(store_path_train): os.makedirs(store_path_train) if not os.path.isdir(store_path_test): os.makedirs(store_path_test) if datasets == 'cifar10': if attack == 'fgsm': step_size = [0.01, 0.02, 0.03] for s in range(0, len(step_size)): samples_path = train_path + '/' + str(step_size[s]) [image_list, image_files, real_labels, predicted_labels] = get_data_file(samples_path) ind = [] for i in range(len(image_list)): ind.append(image_files[i].split('_')[0]) for i in range(0, int(math.ceil(X_test.shape[0] / 6))): if str(i) in ind: i_index = ind.index(str(i)) image_files[i_index] = str( step_size[s]) + '_' + image_files[i_index] test_p = store_path_test + '/' + image_files[i_index] np.save(test_p, image_list[i_index]) for i in range(int(math.ceil(X_test.shape[0] / 6)), X_test.shape[0]): if str(i) in ind: i_index = ind.index(str(i)) image_files[i_index] = str( step_size[s]) + '_' + image_files[i_index] train_p = store_path_train + '/' + image_files[i_index] np.save(train_p, image_list[i_index]) ''' for i in range(0, math.ceil(len(all_images)/6)): test_p =store_path_test+'/'+all_image_files[i] np.save(test_p, all_images[i]) for i in range(math.ceil(len(all_images)/6), len(all_images)): train_p =store_path_train+'/'+all_image_files[i] np.save(train_p, all_images[i]) ''' if attack == 'cw': targets = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] cw_ini_cons = [0.1, 0.2, 0.3] for t in range(0, len(targets)): for c in range(0, len(cw_ini_cons)): samples_path = train_path + '/' + str( cw_ini_cons[c]) + '_' + str(targets[t]) [image_list, image_files, real_labels, predicted_labels] = get_data_file(samples_path) ind = [] for i in range(len(image_list)): ind.append(image_files[i].split('_')[0]) for i in range(1000 * t, 1000 * t + int(math.ceil(1000 / 6))): if str(i) in ind: i_index = ind.index(str(i)) image_files[i_index] = str( cw_ini_cons[c]) + '_' + image_files[i_index] test_p = store_path_test + '/' + image_files[ i_index] np.save(test_p, image_list[i_index]) for i in range(1000 * t + int(math.ceil(1000 / 6)), 1000 * (t + 1)): if str(i) in ind: i_index = ind.index(str(i)) image_files[i_index] = str( cw_ini_cons[c]) + '_' + image_files[i_index] train_p = store_path_train + '/' + image_files[ i_index] np.save(train_p, image_list[i_index]) ''' for i in range(0, math.ceil(X_test.shape[0]/(len(targets)*len(cw_ini_cons)))): train_p =store_path_train+'/'+str(cw_ini_cons[c])+'_'+str(targets[t])+'_'+image_files[i] #imsave(train_p, image_list[i]) np.save(train_p, image_list[i]) j=i+len(image_list)-math.ceil(X_test.shape[0]/(len(targets)*len(cw_ini_cons))) #print(j) test_p =store_path_test+'/'+str(cw_ini_cons[c])+'_'+str(targets[t])+'_'+image_files[j] #imsave(test_p, image_list[j]) np.save(test_p, image_list[j]) ''' if attack == 'jsma': targets = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] jsma_var = [0.09, 0.1, 0.11] for t in range(0, len(targets)): for c in range(0, len(jsma_var)): samples_path = train_path + '/' + str( jsma_var[c]) + '_' + str(targets[t]) [image_list, image_files, real_labels, predicted_labels] = get_data_file(samples_path) ind = [] for i in range(len(image_list)): ind.append(image_files[i].split('_')[0]) for i in range(1000 * t, 1000 * t + int(math.ceil(1000 / 6))): if str(i) in ind: i_index = ind.index(str(i)) image_files[i_index] = str( jsma_var[c]) + '_' + image_files[i_index] test_p = store_path_test + '/' + image_files[ i_index] np.save(test_p, image_list[i_index]) for i in range(1000 * t + int(math.ceil(1000 / 6)), 1000 * (t + 1)): if str(i) in ind: i_index = ind.index(str(i)) image_files[i_index] = str( jsma_var[c]) + '_' + image_files[i_index] train_p = store_path_train + '/' + image_files[ i_index] np.save(train_p, image_list[i_index]) ''' for i in range(0, math.ceil(X_test.shape[0]/(len(targets)*len(jsma_var)))): train_p =store_path_train+'/'+str(jsma_var[c])+'_'+str(targets[t])+'_'+image_files[i] #imsave(train_p, image_list[i]) np.save(train_p, image_list[i]) j=i+len(image_list)-math.ceil(X_test.shape[0]/(len(targets)*len(jsma_var))) #print(j) test_p =store_path_test+'/'+str(jsma_var[c])+'_'+str(targets[t])+'_'+image_files[j] #imsave(test_p, image_list[j]) np.save(test_p, image_list[j]) ''' if datasets == 'mnist': if attack == 'fgsm': step_size = [0.2, 0.3, 0.4] for s in range(0, len(step_size)): samples_path = train_path + '/' + str(step_size[s]) [image_list, image_files, real_labels, predicted_labels] = get_data_file(samples_path) ind = [] for i in range(len(image_list)): ind.append(image_files[i].split('_')[0]) for i in range(0, int(math.ceil(X_test.shape[0] / 7))): if str(i) in ind: i_index = ind.index(str(i)) image_files[i_index] = str( step_size[s]) + '_' + image_files[i_index] test_p = store_path_test + '/' + image_files[i_index] np.save(test_p, image_list[i_index]) for i in range(int(math.ceil(X_test.shape[0] / 7)), X_test.shape[0]): if str(i) in ind: i_index = ind.index(str(i)) image_files[i_index] = str( step_size[s]) + '_' + image_files[i_index] train_p = store_path_train + '/' + image_files[i_index] np.save(train_p, image_list[i_index]) if attack == 'cw': targets = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] cw_ini_cons = [9, 10, 11] for t in range(0, len(targets)): for c in range(0, len(cw_ini_cons)): samples_path = train_path + '/' + str( cw_ini_cons[c]) + '_' + str(targets[t]) [image_list, image_files, real_labels, predicted_labels] = get_data_file(samples_path) ind = [] for i in range(len(image_list)): ind.append(image_files[i].split('_')[0]) for i in range(1000 * t, 1000 * t + int(math.ceil(1000 / 7))): if str(i) in ind: i_index = ind.index(str(i)) image_files[i_index] = str( cw_ini_cons[c]) + '_' + image_files[i_index] test_p = store_path_test + '/' + image_files[ i_index] np.save(test_p, image_list[i_index]) for i in range(1000 * t + int(math.ceil(1000 / 7)), 1000 * (t + 1)): if str(i) in ind: i_index = ind.index(str(i)) image_files[i_index] = str( cw_ini_cons[c]) + '_' + image_files[i_index] train_p = store_path_train + '/' + image_files[ i_index] np.save(train_p, image_list[i_index]) if attack == 'jsma': targets = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] jsma_var = [0.09, 0.1, 0.11] for t in range(0, len(targets)): for c in range(0, len(jsma_var)): samples_path = train_path + '/' + str( jsma_var[c]) + '_' + str(targets[t]) [image_list, image_files, real_labels, predicted_labels] = get_data_file(samples_path) ind = [] for i in range(len(image_list)): ind.append(image_files[i].split('_')[0]) for i in range(1000 * t, 1000 * t + int(math.ceil(1000 / 7))): if str(i) in ind: i_index = ind.index(str(i)) image_files[i_index] = str( jsma_var[c]) + '_' + image_files[i_index] test_p = store_path_test + '/' + image_files[ i_index] np.save(test_p, image_list[i_index]) for i in range(1000 * t + int(math.ceil(1000 / 7)), 1000 * (t + 1)): if str(i) in ind: i_index = ind.index(str(i)) image_files[i_index] = str( jsma_var[c]) + '_' + image_files[i_index] train_p = store_path_train + '/' + image_files[ i_index] np.save(train_p, image_list[i_index])
def ric(datasets, model, samples_path, quality, epoch=49): """ :param datasets :param model :param samples_path :return: """ # Object used to keep track of (and return) key accuracies sess, preds, x, y, model, feed_dict = model_load(datasets, model, epoch=epoch) [image_list, image_files, real_labels, predicted_labels] = get_data_file(samples_path) ori_path = samples_path.replace('test_data', 'ric_ori') if not os.path.exists(ori_path): os.makedirs(ori_path) ic_path = samples_path.replace('test_data', 'ric_ic') if not os.path.exists(ic_path): os.makedirs(ic_path) count = 0 for i in range(len(image_list)): #jj=np.asarray(image_list[i:i+1]) #print(jj.shape) if datasets == 'mnist': adv_img_deprocessed = deprocess_image_1( np.asarray(image_list[i:i + 1]))[0] elif datasets == 'cifar10': adv_img_deprocessed = deprocess_image_1( np.asarray(image_list[i:i + 1])) saved_adv_image_path = os.path.join( ori_path, image_files[i].replace("npy", "png")) imsave(saved_adv_image_path, adv_img_deprocessed) output_IC_path = os.path.join(ic_path, image_files[i].replace("npy", "jpg")) cmd = '../../guetzli/bin/Release/guetzli --quality {} {} {}'.format( quality, saved_adv_image_path, output_IC_path) assert os.system( cmd ) == 0, 'guetzli tool should be install before, https://github.com/google/guetzli' if datasets == 'cifar10': IC_image = Image.open(output_IC_path).convert('RGB') IC_image = np.asarray( [np.array(IC_image).astype('float32') / 255.0]) #IC_image=IC_image.reshape(32, 32, 3) elif datasets == 'mnist': IC_image = Image.open(output_IC_path).convert('L') IC_image = np.expand_dims(np.array(IC_image).astype('float32'), axis=0) / 255.0 IC_image = IC_image.reshape(-1, 28, 28, 1) if model_argmax(sess, x, preds, IC_image, feed=feed_dict) != int( real_labels[i]): count = count + 1 result = 1.0 * count / len(image_list) print('Robustness to image compression is %.4f' % (result)) # Close TF session sess.close() return result
def mcdc(datasets, model_name, samples_path, de='False', attack='fgsm', just_adv=False, epoch=9): X_train, Y_train, X_test, Y_test = get_data(datasets) samples = X_test if samples_path not in ['test']: if not just_adv: [image_list, _, _, _] = get_data_file(samples_path) samples_adv = np.asarray(image_list) samples = np.concatenate((samples, samples_adv)) print("Combine data") else: [image_list, _, _, _] = get_data_file(samples_path) samples = np.asarray(image_list) print("Just adv") tf.reset_default_graph() sess, preds, x, y, model, feed_dict = model_load(datasets=datasets, model_name=model_name, de=de, attack=attack, epoch=epoch) # dict = model.fprop(x) layer_names = model.layer_names sess.close() del sess, preds, x, y, model, feed_dict gc.collect() l = 0 ss = [] sc_pr = [] neuron = [] for key in layer_names: #model.layer_names: if 'ReLU' in key or 'probs' in key: print(l) tf.reset_default_graph() sess, preds, x, y, model, feed_dict = model_load( datasets=datasets, model_name=model_name, de=de, attack=attack, epoch=epoch) dict = model.fprop(x) tensor = dict[key] neuron.append(tensor.shape[-1]) layer_output = [] n_batches = int(np.ceil(1.0 * samples.shape[0] / 256)) for batch in range(n_batches): start = batch * 256 end = np.minimum(len(samples), (batch + 1) * 256) feed = {x: samples[start:end]} if feed_dict is not None: feed.update(feed_dict) # v = sess.run(tensor, feed_dict=feed) layer_output = layer_output + sess.run( tensor, feed_dict=feed).tolist() sess.close() del sess, preds, x, y, model, feed_dict gc.collect() layer_output = np.asarray(layer_output) layer_sign = np.zeros( (layer_output.shape[0], layer_output.shape[-1])) for num in range(len(layer_output)): for num_neuron in xrange(layer_output[num].shape[-1]): if np.mean(layer_output[num][..., num_neuron]) > 0.0: layer_sign[num][num_neuron] = 1 del layer_output gc.collect() if l == 0: for i in range(len(samples) - 1): temp = [] for j in range(i + 1, len(samples)): sc = xor(layer_sign[i], layer_sign[j]) if len(sc) > 1: sc = [] temp.append(sc) sc_pr.append(temp) else: for i in range(len(samples) - 1): for j in range(i + 1, len(samples)): sc = xor(layer_sign[i], layer_sign[j]) if len(sc_pr[i][j - i - 1]) == 1: n_pr = str(l) + '_' + str(sc_pr[i][j - i - 1]) for n in sc: n = str(l + 1) + '_' + str(n) combination = tuple([n_pr, n]) if combination not in ss: # ss.append(tuple([n_pr, n])) ss.append(combination) if len(sc) > 1: sc = [] sc_pr[i][j - i - 1] = sc l = l + 1 total = 0 for i in range(len(neuron) - 1): total = total + int(neuron[i]) * int(neuron[i + 1]) print(1.0 * len(set(ss)) / total) return 1.0 * len(set(ss)) / total
def ct(datasets, model_name, samples_path, t=2, p=0.5, de='False', attack='fgsm', just_adv=False, epoch=9): X_train, Y_train, X_test, Y_test = get_data(datasets) samples = X_test if samples_path not in ['test']: if not just_adv: [image_list, _, _, _] = get_data_file( samples_path) #image_files, real_labels, predicted_labels samples_adv = np.asarray(image_list) samples = np.concatenate((samples, samples_adv)) print("Combine data") else: [image_list, _, _, _] = get_data_file( samples_path) #image_files, real_labels, predicted_labels samples = np.asarray(image_list) print("Just adv") tf.reset_default_graph() sess, preds, x, y, model, feed_dict = model_load(datasets=datasets, model_name=model_name, de=de, attack=attack, epoch=epoch) layers_combination = neuron_combination(t, model, x) sess.close() del sess, preds, x, y, model, feed_dict gc.collect() n_batches = int(np.ceil(1.0 * samples.shape[0] / 512)) for num in range(n_batches): print(num) start = num * 512 end = np.minimum(len(samples), (num + 1) * 512) batch_samples = samples[start:end] tf.reset_default_graph() sess, preds, x, y, model, feed_dict = model_load(datasets=datasets, model_name=model_name, de=de, attack=attack, epoch=epoch) layers_activation = cal_activation(sess, x, batch_samples, model, feed_dict) sess.close() del sess, preds, x, y, model, feed_dict, batch_samples gc.collect() layers_combination = update_combination(layers_combination, layers_activation, t) sparse = 0 dense = 0 p_completeness = 0 total = 0 t_t = pow(2, t) completeness = 1.0 * t_t * p for layer in layers_combination: total += len(layer) for combination in layer: s = sum(combination) dense += s if s >= completeness: p_completeness += 1 if combination == np.ones(t_t).astype('int').tolist(): sparse += 1 sparse_coverage = 1.0 * sparse / total dense_coverage = 1.0 * dense / (t_t * total) pt_completeness = 1.0 * p_completeness / total print([sparse_coverage, dense_coverage, pt_completeness]) return [sparse_coverage, dense_coverage, pt_completeness]
def neuron_coverage(datasets, model_name, samples_path, others='', train_num=0, test_num=0, de=False, attack='fgsm', just_adv=False, epoch=49, datasettype='test'): """ :param datasets :param model :param samples_path :return: """ # Object used to keep track of (and return) key accuracies X_train, Y_train, X_test, Y_test = get_data(datasets) if datasettype == 'train': samples = X_train[:train_num] else: samples = X_test[:test_num] if samples_path not in ['test']: if not just_adv: [image_list, image_files, real_labels, predicted_labels] = get_data_file(samples_path) #samples_adv = np.asarray([preprocess_image_1(image.astype('float64')) for image in image_list]) samples_adv = np.asarray(image_list) samples = np.concatenate((samples, samples_adv)) print("Combine data") else: [image_list, image_files, real_labels, predicted_labels] = get_data_file(samples_path) #samples_adv = np.asarray([preprocess_image_1(image.astype('float64')) for image in image_list]) samples = np.asarray(image_list) #samples = np.concatenate((samples, samples_adv)) print("Just adv") tf.reset_default_graph() sess, preds, x, y, model, feed_dict = model_load(datasets=datasets, model_name=model_name, de=de, attack=attack, epoch=epoch, others=others) model_layer_dict = init_coverage_tables(model) sess.close() del sess, preds, x, y, model, feed_dict gc.collect() #ceil取整数 n_batches = int(np.ceil(samples.shape[0] / 256)) for i in range(n_batches): print(i) start = i * 256 end = np.minimum(len(samples), (i + 1) * 256) tf.reset_default_graph() sess, preds, x, y, model, feed_dict = model_load(datasets=datasets, model_name=model_name, de=de, attack=attack, epoch=epoch, others=others) model_layer_dict = update_coverage(sess, x, samples[start:end], model, model_layer_dict, feed_dict, threshold=0) sess.close() del sess, preds, x, y, model, feed_dict gc.collect() result = neuron_covered(model_layer_dict)[2] print('covered neurons percentage %d neurons %.4f' % (len(model_layer_dict), result)) if result >= 0.999: break return len(model_layer_dict), result
def sc(datasets, model_name, samples_path, layer=-3, num_section=1000, de='False', attack='fgsm', epoch=9): X_train, Y_train, X_test, Y_test = get_data(datasets) if de == True: adv_train_path = '../adv_result/' + datasets + '/' + attack + '/' + model_name + '/train_data' [image_list_train, _, real_labels, _] = get_data_file(adv_train_path) samples_train = np.asarray(image_list_train) if datasets == "mnist": indexs = random.sample(range(len(samples_train)), int(len(samples_train)/12)) elif datasets == "cifar10": indexs = random.sample(range(len(samples_train)), int(len(samples_train) / 10)) train_samples = np.concatenate((samples_train[indexs], X_train[:5000])) store_path = "../suprise/" + datasets + "/" + model_name + "/" + attack + '/' else: train_samples = X_train[:5000] store_path = "../suprise/" + datasets + "/" + model_name + "/ori/" if not os.path.exists(store_path): os.makedirs(store_path) a_n_train = [] train_labels = [] n_batches = int(np.ceil(1.0 * train_samples.shape[0] / 512)) for num in range(n_batches): print(num) start = num * 512 end = np.minimum(len(train_samples), (num + 1) * 512) tf.reset_default_graph() sess, preds, x, y, model, feed_dict = model_load(datasets=datasets, model_name=model_name, de=de, attack=attack, epoch=epoch) train_labels = train_labels+model_argmax(sess, x, preds, train_samples[start:end], feed_dict).tolist() a_n_train = a_n_train+at_training(sess, x, train_samples[start:end], model, feed_dict, layer).tolist() sess.close() del sess, preds, x, y, model, feed_dict gc.collect() a_n_train = np.asarray(a_n_train) train_labels = np.asarray(train_labels) np.save(store_path + "a_n_train.npy", a_n_train) np.save(store_path + "train_labels.npy", train_labels) else: a_n_train = np.load(store_path + "a_n_train.npy") train_labels = np.load(store_path + "train_labels.npy") class_inds = {} for i in range(10): class_inds[i] = np.where(train_labels == i)[0] kdes_store_path=store_path+'kdes.npy' if os.path.exists(kdes_store_path): kdes = np.load(kdes_store_path).item() else: kdes = {} for i in range(10): scott_bw = pow(len(a_n_train[class_inds[i]]), -1.0/(len(a_n_train[0])+4)) kdes[i] = KernelDensity(kernel='gaussian', bandwidth=scott_bw).fit(a_n_train[class_inds[i]]) np.save(kdes_store_path, kdes) lsa = [] dsa = [] c = set(range(len(train_labels))) lsa_test_store_path=store_path+"lsa_test.npy" dsa_test_store_path=store_path+"dsa_test.npy" if os.path.exists(lsa_test_store_path) and os.path.exists(dsa_test_store_path): lsa=np.load(lsa_test_store_path).tolist() dsa=np.load(dsa_test_store_path).tolist() else: # X_test=X_test n_batches = int(np.ceil(1.0 * X_test.shape[0] / 512)) for num in range(n_batches): print(num) start = num * 512 end = np.minimum(len(X_test), (num + 1) * 512) batch_samples = X_test[start:end] tf.reset_default_graph() sess, preds, x, y, model, feed_dict = model_load(datasets=datasets, model_name=model_name, de=de, attack=attack, epoch=epoch) batch_labels = model_argmax(sess, x, preds, batch_samples, feed=feed_dict) a_n_test = at_training(sess, x, batch_samples, model, feed_dict, layer) sess.close() del sess, preds, x, y, model, feed_dict gc.collect() for i in range(len(batch_samples)): kd_value = kdes[batch_labels[i]].score_samples(np.reshape(a_n_test[i],(1,-1)))[0] #/ len(a_n_train[class_inds[batch_labels[i]]]) lsa.append(-kd_value) data = np.asarray([a_n_test[i]], dtype=np.float32) batch = np.asarray(a_n_train[class_inds[batch_labels[i]]], dtype=np.float32) # dist = np.linalg.norm(data - batch, axis=1) dist = cdist(data, batch)[0] dist_a = min(dist) alpha_a = np.asarray([batch[np.argmin(dist)]]) c_i = set(class_inds[batch_labels[i]]) c_ni = list(c^c_i) batch = np.asarray(a_n_train[c_ni], dtype=np.float32) # dist_b = min(np.linalg.norm(alpha_a - batch, axis=1)) dist_b = min(cdist(alpha_a, batch)[0]) dsa.append(dist_a / dist_b) np.save(store_path + "lsa_test.npy", np.asarray(lsa)) np.save(store_path + "dsa_test.npy", np.asarray(dsa)) upper_lsa_test=max(lsa) lower_lsa_test = min(lsa) upper_dsa_test=max(dsa) lower_dsa_test = min(dsa) if samples_path not in ['test']: lsa = [] dsa = [] [image_list, _, _, _] = get_data_file(samples_path) # image_files, real_labels, predicted_labels samples_adv = np.asarray(image_list) n_batches = int(np.ceil(1.0 * samples_adv.shape[0] / 512)) for num in range(n_batches): print(num) start = num * 512 end = np.minimum(len(samples_adv), (num + 1) * 512) batch_samples = samples_adv[start:end] tf.reset_default_graph() sess, preds, x, y, model, feed_dict = model_load(datasets=datasets, model_name=model_name, de=de, attack=attack, epoch=epoch) batch_labels = model_argmax(sess, x, preds, batch_samples, feed=feed_dict) a_n_adv = at_training(sess, x, batch_samples, model, feed_dict, layer) sess.close() del sess, preds, x, y, model, feed_dict gc.collect() for i in range(len(batch_samples)): kd_value = kdes[batch_labels[i]].score_samples(np.reshape(a_n_adv[i],(1,-1)))[0] #/ len(a_n_train[class_inds[batch_labels[i]]]) lsa.append(-kd_value) data = np.asarray([a_n_adv[i]], dtype=np.float32) batch = np.asarray(a_n_train[class_inds[batch_labels[i]]], dtype=np.float32) # dist = np.linalg.norm(data - batch, axis=1) dist = cdist(data, batch)[0] dist_a = min(dist) alpha_a = np.asarray([batch[np.argmin(dist)]]) c_i = set(class_inds[batch_labels[i]]) c_ni = list(c ^ c_i) batch = np.asarray(a_n_train[c_ni], dtype=np.float32) # dist_b = min(np.linalg.norm(alpha_a - batch, axis=1)) dist_b = min(cdist(alpha_a, batch)[0]) dsa.append(dist_a / dist_b) for i in range(len(lsa)): lsa[i]=(lsa[i]-lower_lsa_test)/(upper_lsa_test-lower_lsa_test) dsa[i]=(dsa[i]-lower_dsa_test)/(upper_dsa_test-lower_dsa_test) lsa_mean = np.mean(lsa) lsa_std = np.std(lsa) dsa_mean = np.mean(dsa) dsa_std = np.std(dsa) half_section = int(num_section / 2) n_section_lsa=np.zeros(num_section).astype('int64') n_section_dsa=np.zeros(num_section).astype('int64') for i in range(len(lsa)): l = lsa[i]*half_section d = dsa[i]*half_section if math.ceil(l) < num_section and math.floor(l) >= 0: if math.ceil(l) == math.floor(l): n_section_lsa[int(l)-1]=1 else: n_section_lsa[int(l)]=1 if math.ceil(d) < num_section and math.floor(d) >= 0: if math.ceil(d) == math.floor(d): n_section_dsa[int(d)-1]=1 else: n_section_dsa[int(d)]=1 cov_lsa_1=1.0 * sum(n_section_lsa[:half_section])/(half_section) cov_dsa_1=1.0 * sum(n_section_dsa[:half_section])/(half_section) cov_lsa_2 = 1.0 * sum(n_section_lsa[half_section:]) / (half_section) cov_dsa_2 = 1.0 * sum(n_section_dsa[half_section:]) / (half_section) print([lsa_mean, lsa_std, cov_lsa_1, cov_lsa_2, upper_lsa_test,lower_lsa_test, dsa_mean, dsa_std, cov_dsa_1, cov_dsa_2, upper_dsa_test,lower_dsa_test]) return [lsa_mean, lsa_std, cov_lsa_1, cov_lsa_2, upper_lsa_test,lower_lsa_test, dsa_mean, dsa_std, cov_dsa_1, cov_dsa_2, upper_dsa_test,lower_dsa_test]
def multi_testing_criteria(datasets, model_name, samples_path, others='', train_num=0, test_num=0, std_range=0.0, k_n=1000, k_l=2, de=False, attack='fgsm', just_adv=False, epoch=4, datasettype='test'): """ :param datasets :param model :param samples_path :param std_range :param k_n :param k_l :return: """ X_train, Y_train, X_test, Y_test = get_data(datasets) X_train = X_train[:train_num] if datasettype == 'train': samples = X_train[:train_num] else: samples = X_test[:test_num] if samples_path not in ['test']: if not just_adv: [image_list, _, _, _] = get_data_file(samples_path) samples_adv = np.asarray(image_list) samples = np.concatenate((samples, samples_adv)) print("Combine data") else: [image_list, _, _, _] = get_data_file(samples_path) samples = np.asarray(image_list) print("Just adv") if de == True: train_boundary_path = '../adv_result/' + datasets + '/' + attack + '/' + model_name + '/train_data' [image_list_train, _, _, _] = get_data_file(train_boundary_path) samples_train = np.asarray(image_list_train) X_train_boundary = np.concatenate((samples_train, X_train)) store_path = "../multi_testing_criteria/" + datasets + "/" + model_name + "/" + attack + '/' else: X_train_boundary = X_train store_path = "../multi_testing_criteria/" + datasets + "/" + model_name + "/ori/" if not os.path.exists(store_path): os.makedirs(store_path) tf.reset_default_graph() sess, preds, x, y, model, feed_dict = model_load(datasets=datasets, model_name=model_name, de=de, attack=attack, epoch=epoch, others=others) boundary = neuron_boundary(sess, x, X_train_boundary, model, feed_dict) sess.close() del sess, preds, x, y, model, feed_dict gc.collect() np.save(store_path + "boundary.npy", np.asarray(boundary)) else: boundary = np.load(store_path + "boundary.npy").tolist() k_coverage, boundary_coverage, neuron_number = init_coverage_metric( boundary, k_n) if samples_path == 'test': if datasettype == 'train': store_path = store_path + 'train/' else: store_path = store_path + 'test/' else: if datasettype == 'train': store_path = store_path + 'train_adv' + samples_path.split( '/')[-3] + '/' else: store_path = store_path + 'test_adv' + samples_path.split( '/')[-3] + '/' if not os.path.exists(store_path): cal = True os.makedirs(store_path) else: cal = False NP = [] n_batches = int(np.ceil(1.0 * samples.shape[0] / 256)) for num in range(n_batches): print(num) start = num * 256 end = np.minimum(len(samples), (num + 1) * 256) if cal: input_data = samples[start:end] tf.reset_default_graph() sess, preds, x, y, model, feed_dict = model_load( datasets=datasets, model_name=model_name, de=de, attack=attack, epoch=epoch, others=others) layers_output = calculate_layers(sess, x, model, feed_dict, input_data, store_path, num) sess.close() del sess, preds, x, y, model, feed_dict, input_data gc.collect() else: layers_output = np.load(store_path + 'layers_output_' + str(num) + '.npy') k_coverage, boundary_coverage = update_multi_coverage_neuron( layers_output, k_n, boundary, k_coverage, boundary_coverage, std_range) layer_coverage = calculate_coverage_layer(layers_output, k_l, end - start) if num == 0: layer = [set([])] * layer_coverage.shape[0] for i in range(len(layer_coverage)): for j in range(len(layer_coverage[i])): # |表示按位运算 layer[i] = layer[i] | layer_coverage[i][j] sample_coverage = np.transpose(layer_coverage, (1, 0)) for i in range(len(sample_coverage)): sc = sample_coverage[i].tolist() if sc not in NP: NP.append(sc) del layers_output gc.collect() KMN = 0 NB = 0 SNA = 0 for i in range(len(k_coverage)): for j in range(len(k_coverage[i])): for t in range(len(k_coverage[i][j])): if k_coverage[i][j][t] > 0: KMN += 1 if boundary_coverage[i][j][1] > 0: NB += 1 SNA += 1 if boundary_coverage[i][j][0] > 0: NB += 1 KMN = 1.0 * KMN / (k_n * neuron_number) NB = 1.0 * NB / (2 * neuron_number) SNA = 1.0 * SNA / neuron_number TKNC = sum(len(neurons) for neurons in layer) TKNC = 1.0 * TKNC / neuron_number TKNP = len(NP) print([KMN, NB, SNA, TKNC, TKNP]) return [KMN, NB, SNA, TKNC, TKNP]
def choose_mu(attack='fgsm', datasets='mnist', total_num=10000, model_name='lenet1', mu_var='gf'): tf.reset_default_graph() tf.set_random_seed(1234) config = tf.ConfigProto() #config.gpu_options.per_process_gpu_memory_fraction = 0.7 config.gpu_options.allow_growth = True sess = tf.Session(config=config) X_train, Y_train, X_test, Y_test = get_data(datasets) sess, preds, x, y, model, feed_dict = model_load(datasets, model_name, de=False, epoch=9, attack='fgsm', mu=True, mu_var=mu_var) pre = model_prediction(sess, x, preds, X_test, feed=feed_dict, datasets=datasets) acc_pre_index = [] for i in range(0, pre.shape[0]): if np.argmax(pre[i]) == np.argmax(Y_test[i]): acc_pre_index.append(i) input_shape, nb_classes = get_shape(datasets) train_path = '../adv_result/' + datasets + '/' + attack + '/' + model_name store_path_train = '../adv_result/mu_' + datasets + '/' + mu_var + '/' + attack + '/' + model_name + '/train_data' store_path_test = '../adv_result/mu_' + datasets + '/' + mu_var + '/' + attack + '/' + model_name + '/test_data' if not os.path.isdir(store_path_train): os.makedirs(store_path_train) if not os.path.isdir(store_path_test): os.makedirs(store_path_test) if datasets == 'cifar10': if attack == 'fgsm': step_size = [0.01, 0.02, 0.03] for s in range(0, len(step_size)): samples_path = train_path + '/' + str(step_size[s]) [image_list, image_files, real_labels, predicted_labels] = get_data_file(samples_path) samples_adv = np.asarray(image_list) result = model_prediction(sess, x, preds, samples_adv, feed=feed_dict, datasets=datasets) ind_file = [] for i in range(len(image_list)): ind_file.append(image_files[i].split('_')[0]) ind = [] for i in range(len(image_list)): nn = int(image_files[i].split('_')[0]) if (nn in acc_pre_index) and (predicted_labels[i] == np.argmax(result[i])): ind.append(image_files[i].split('_')[0]) for i in range(0, int(math.ceil(X_test.shape[0] / 6))): if str(i) in ind: i_index = ind_file.index(str(i)) image_files[i_index] = str( step_size[s]) + '_' + image_files[i_index] test_p = store_path_test + '/' + image_files[i_index] np.save(test_p, image_list[i_index]) for i in range(int(math.ceil(X_test.shape[0] / 6)), X_test.shape[0]): if str(i) in ind: i_index = ind_file.index(str(i)) image_files[i_index] = str( step_size[s]) + '_' + image_files[i_index] train_p = store_path_train + '/' + image_files[i_index] np.save(train_p, image_list[i_index]) if attack == 'cw': targets = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] cw_ini_cons = [0.1, 0.2, 0.3] for t in range(0, len(targets)): for c in range(0, len(cw_ini_cons)): samples_path = train_path + '/' + str( cw_ini_cons[c]) + '_' + str(targets[t]) [image_list, image_files, real_labels, predicted_labels] = get_data_file(samples_path) samples_adv = np.asarray(image_list) result = model_prediction(sess, x, preds, samples_adv, feed=feed_dict, datasets=datasets) ind_file = [] for i in range(len(image_list)): ind_file.append(image_files[i].split('_')[0]) ind = [] for i in range(len(image_list)): nn = int(image_files[i].split('_')[0]) if (nn in acc_pre_index) and (predicted_labels[i] == np.argmax(result[i])): ind.append(image_files[i].split('_')[0]) for i in range(1000 * t, 1000 * t + int(math.ceil(1000 / 6))): if str(i) in ind: i_index = ind_file.index(str(i)) image_files[i_index] = str( cw_ini_cons[c]) + '_' + image_files[i_index] test_p = store_path_test + '/' + image_files[ i_index] np.save(test_p, image_list[i_index]) for i in range(1000 * t + int(math.ceil(1000 / 6), 1000 * (t + 1))): if str(i) in ind: i_index = ind_file.index(str(i)) image_files[i_index] = str( cw_ini_cons[c]) + '_' + image_files[i_index] train_p = store_path_train + '/' + image_files[ i_index] np.save(train_p, image_list[i_index]) if attack == 'jsma': targets = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] jsma_var = [0.09, 0.1, 0.11] for t in range(0, len(targets)): for c in range(0, len(jsma_var)): samples_path = train_path + '/' + str( jsma_var[c]) + '_' + str(targets[t]) [image_list, image_files, real_labels, predicted_labels] = get_data_file(samples_path) samples_adv = np.asarray(image_list) result = model_prediction(sess, x, preds, samples_adv, feed=feed_dict, datasets=datasets) ind_file = [] for i in range(len(image_list)): ind_file.append(image_files[i].split('_')[0]) ind = [] for i in range(len(image_list)): nn = int(image_files[i].split('_')[0]) if (nn in acc_pre_index) and (predicted_labels[i] == np.argmax(result[i])): ind.append(image_files[i].split('_')[0]) for i in range(1000 * t, 1000 * t + int(math.ceil(1000 / 6))): if str(i) in ind: i_index = ind_file.index(str(i)) image_files[i_index] = str( jsma_var[c]) + '_' + image_files[i_index] test_p = store_path_test + '/' + image_files[ i_index] np.save(test_p, image_list[i_index]) for i in range(1000 * t + int(math.ceil(1000 / 6)), 1000 * (t + 1)): if str(i) in ind: i_index = ind_file.index(str(i)) image_files[i_index] = str( jsma_var[c]) + '_' + image_files[i_index] train_p = store_path_train + '/' + image_files[ i_index] np.save(train_p, image_list[i_index]) if datasets == 'mnist': if attack == 'fgsm': step_size = [0.2, 0.3, 0.4] for s in range(0, len(step_size)): samples_path = train_path + '/' + str(step_size[s]) [image_list, image_files, real_labels, predicted_labels] = get_data_file(samples_path) samples_adv = np.asarray(image_list) result = model_prediction(sess, x, preds, samples_adv, feed=feed_dict, datasets=datasets) ind_file = [] for i in range(len(image_list)): ind_file.append(image_files[i].split('_')[0]) ind = [] for i in range(len(image_list)): nn = int(image_files[i].split('_')[0]) if (nn in acc_pre_index) and (predicted_labels[i] == np.argmax(result[i])): ind.append(image_files[i].split('_')[0]) for i in range(0, int(math.ceil(X_test.shape[0] / 7))): if str(i) in ind: i_index = ind_file.index(str(i)) image_files[i_index] = str( step_size[s]) + '_' + image_files[i_index] test_p = store_path_test + '/' + image_files[i_index] np.save(test_p, image_list[i_index]) for i in range(int(math.ceil(X_test.shape[0] / 7)), X_test.shape[0]): if str(i) in ind: i_index = ind_file.index(str(i)) image_files[i_index] = str( step_size[s]) + '_' + image_files[i_index] train_p = store_path_train + '/' + image_files[i_index] np.save(train_p, image_list[i_index]) if attack == 'cw': targets = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] cw_ini_cons = [9, 10, 11] for t in range(0, len(targets)): for c in range(0, len(cw_ini_cons)): samples_path = train_path + '/' + str( cw_ini_cons[c]) + '_' + str(targets[t]) [image_list, image_files, real_labels, predicted_labels] = get_data_file(samples_path) samples_adv = np.asarray(image_list) result = model_prediction(sess, x, preds, samples_adv, feed=feed_dict, datasets=datasets) ind_file = [] for i in range(len(image_list)): ind_file.append(image_files[i].split('_')[0]) ind = [] for i in range(len(image_list)): nn = int(image_files[i].split('_')[0]) if (nn in acc_pre_index) and (predicted_labels[i] == np.argmax(result[i])): ind.append(image_files[i].split('_')[0]) for i in range(1000 * t, 1000 * t + int(math.ceil(1000 / 7))): if str(i) in ind: i_index = ind_file.index(str(i)) image_files[i_index] = str( cw_ini_cons[c]) + '_' + image_files[i_index] test_p = store_path_test + '/' + image_files[ i_index] np.save(test_p, image_list[i_index]) for i in range(1000 * t + int(math.ceil(1000 / 7)), 1000 * (t + 1)): if str(i) in ind: i_index = ind_file.index(str(i)) image_files[i_index] = str( cw_ini_cons[c]) + '_' + image_files[i_index] train_p = store_path_train + '/' + image_files[ i_index] np.save(train_p, image_list[i_index]) if attack == 'jsma': targets = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] jsma_var = [0.09, 0.1, 0.11] for t in range(0, len(targets)): for c in range(0, len(jsma_var)): samples_path = train_path + '/' + str( jsma_var[c]) + '_' + str(targets[t]) [image_list, image_files, real_labels, predicted_labels] = get_data_file(samples_path) samples_adv = np.asarray(image_list) result = model_prediction(sess, x, preds, samples_adv, feed=feed_dict, datasets=datasets) ind_file = [] for i in range(len(image_list)): ind_file.append(image_files[i].split('_')[0]) ind = [] for i in range(len(image_list)): nn = int(image_files[i].split('_')[0]) if (nn in acc_pre_index) and (predicted_labels[i] == np.argmax(result[i])): ind.append(image_files[i].split('_')[0]) for i in range(1000 * t, 1000 * t + int(math.ceil(1000 / 7))): if str(i) in ind: i_index = ind_file.index(str(i)) image_files[i_index] = str( jsma_var[c]) + '_' + image_files[i_index] test_p = store_path_test + '/' + image_files[ i_index] np.save(test_p, image_list[i_index]) for i in range(1000 * t + int(math.ceil(1000 / 7)), 1000 * (t + 1)): if str(i) in ind: i_index = ind_file.index(str(i)) image_files[i_index] = str( jsma_var[c]) + '_' + image_files[i_index] train_p = store_path_train + '/' + image_files[ i_index] np.save(train_p, image_list[i_index])
def model_training(datasets, model_name, samples_path=None, nb_epochs=6, batch_size=256, learning_rate=0.001, attack=None, mu=False, mu_var='gf'): # Set TF random seed to improve reproducibility tf.set_random_seed(1234) config = tf.ConfigProto() config.gpu_options.allow_growth = True #config.gpu_options.per_process_gpu_memory_fraction = 0.9 # Create TF session and set as Keras backend session sess = tf.Session(config=config) print("Created TensorFlow session.") set_log_level(logging.DEBUG) X_train, Y_train, X_test, Y_test = get_data(datasets) def y_one_hot(label): y = np.zeros(10) y[label] = 1 return y if samples_path != None: [image_list, image_files, real_labels, predicted_labels] = get_data_file(samples_path) #samples_adv = np.asarray([preprocess_image_1(image.astype('float64')) for image in image_list]) samples_adv = np.asarray(image_list) #print(samples_adv.shape) labels_adv = np.asarray( [y_one_hot(int(label)) for label in real_labels]) samples = np.concatenate((X_train, samples_adv)) #print(samples.shape) labels = np.concatenate((Y_train, labels_adv)) if mu == True: model_path = path.mude_model_path + mu_var + '/' + attack + '/' + datasets + "_" + model_name else: model_path = path.de_model_path + attack + '/' + datasets + "_" + model_name else: samples = X_train labels = Y_train model_path = path.model_path + datasets + "_" + model_name input_shape, nb_classes = get_shape(datasets) # Define input TF placeholder x = tf.placeholder(tf.float32, shape=input_shape) y = tf.placeholder(tf.float32, shape=(None, nb_classes)) feed_dict = None model = model_dict[model_name](input_shape, nb_classes) preds = model(x) print("Defined TensorFlow model graph.") ########################################################################### # Training the model using TensorFlow ########################################################################### train_params = { 'nb_epochs': nb_epochs, 'batch_size': batch_size, 'learning_rate': learning_rate, 'train_dir': model_path, 'filename': model_name + '.model' } sess.run(tf.global_variables_initializer()) rng = np.random.RandomState([2017, 8, 30]) model_train(sess, x, y, preds, samples, labels, args=train_params, rng=rng, save=True) # Evaluate the accuracy of the model on legitimate test examples eval_params = {'batch_size': batch_size} accuracy = model_eval(sess, x, y, preds, X_test, Y_test, args=eval_params, feed=feed_dict) print('Test accuracy on legitimate test examples: {0}'.format(accuracy)) #accuracy = model_eval(sess, x, y, preds, X_test, Y_test, args=eval_params, feed=feed_dict) # Close TF session sess.close() print('Finish model training.')