def exp_deep_metric(model_name, use_adv):
rank_lst2 = None
rank_lst2_time = None
input, layers, test, train, pred_test, true_test, pred_test_prob = gen_data(
model_name, use_adv=use_adv, deepxplore=False)
model_path = model_conf.get_model_path(model_conf.fashion, model_name) #
model = load_model(model_path)
start = time.time()
pred_test_prob = model.predict(test)
rank_lst = metrics.deep_metric(pred_test_prob)
end = time.time()
rank_lst_time = start - end
df = pd.DataFrame([])
df['right'] = (pred_test == true_test).astype('int')
df['cam'] = 0
df['cam'].loc[rank_lst] = list(range(1, len(rank_lst) + 1))
df['ctm'] = 0
df['cam_time'] = rank_lst_time
df['ctm_time'] = rank_lst2_time
if rank_lst2 is not None:
df['ctm'].loc[rank_lst2] = list(range(1, len(rank_lst2) + 1))
df['rate'] = 0
if use_adv:
dataset = 'fashion_adv'
else:
dataset = 'fashion'
df.to_csv('./all_output/output_fashion/{}/{}_deep_metric.csv'.format(
model_name, dataset))
def gen_data(model_name, use_adv=True, deepxplore=False):
model_path = model_conf.get_model_path(model_conf.svhn, model_name) #
(X_train, Y_train), (X_test, Y_test) = SVNH_DatasetUtil.load_data()
Y_test = np.argmax(Y_test, axis=1)
# model_path = './model/model_svhn.hdf5'
if use_adv:
attack_lst = ['cw', 'fgsm', 'jsma', 'bim']
adv_image_all = []
adv_label_all = []
for attack in attack_lst:
im, lab = model_conf.get_adv_path(attack, model_conf.svhn, model_name)
adv_image_all.append(np.load(im))
adv_label_all.append(np.load(lab))
adv_image_all = np.concatenate(adv_image_all, axis=0)
adv_label_all = np.concatenate(adv_label_all, axis=0)
print("adv: ", len(adv_image_all))
test = np.concatenate([X_test, adv_image_all], axis=0)
true_test = np.concatenate([Y_test, adv_label_all], axis=0)
else:
test = X_test
true_test = Y_test
train = X_train
model = load_model(model_path)
model.summary()
pred_test_prob = model.predict(test)
pred_test = np.argmax(pred_test_prob, axis=1)
input = model.layers[0].output
# print(dir(model.layers[0]))
# print(input)
if not deepxplore:
if model_name == model_conf.LeNet5:
layers = [model.layers[2].output, model.layers[3].output, model.layers[5].output, model.layers[6].output,
model.layers[8].output, model.layers[9].output, model.layers[10].output]
layers = list(zip(4 * ['conv'] + 3 * ['dense'], layers))
else: # Vgg16
layers = []
for i in range(1, 19):
layers.append(model.layers[i].output)
for i in range(20, 23):
layers.append(model.layers[i].output)
layers = list(zip(18 * ['conv'] + 3 * ['dense'], layers))
else:
if model_name == model_conf.LeNet5:
layers = [model.layers[1].output, model.layers[3].output, model.layers[4].output, model.layers[8].output,
model.layers[8].output, model.layers[9].output, model.layers[10].output]
layers = list(zip(4 * ['conv'] + 3 * ['dense'], layers))
else:
layers = []
for i in range(1, 19):
layers.append(model.layers[i].output)
for i in range(20, 23):
layers.append(model.layers[i].output)
layers = list(zip(18 * ['conv'] + 3 * ['dense'], layers))
# print(len(layers), layers)
return input, layers, test, train, pred_test, true_test, pred_test_prob
def gen_data(model_name, use_adv=True, deepxplore=False):
# path = './fashion-mnist/data/fashion'
# X_train, Y_train = mnist_reader.load_mnist(path, kind='train')
# X_test, Y_test = mnist_reader.load_mnist(path, kind='t10k')
# X_train = X_train.astype('float32').reshape(-1, 28, 28, 1)
# X_test = X_test.astype('float32').reshape(-1, 28, 28, 1)
# X_train /= 255
# X_test /= 255
### modify
(X_train, y_train), (X_test,
Y_test) = fashion_mnist.load_data() ### modify
X_train = X_train.astype('float32').reshape(-1, 28, 28, 1)
X_test = X_test.astype('float32').reshape(-1, 28, 28, 1)
X_train /= 255
X_test /= 255
model_path = model_conf.get_model_path(model_conf.fashion, model_name)
if use_adv:
attack_lst = ['fgsm', 'jsma', 'bim', 'cw']
adv_image_all = []
adv_label_all = []
for attack in attack_lst:
im, lab = model_conf.get_adv_path(attack, model_conf.fashion,
model_name)
adv_image_all.append(np.load(im))
adv_label_all.append(np.load(lab))
adv_image_all = np.concatenate(adv_image_all, axis=0)
adv_label_all = np.concatenate(adv_label_all, axis=0)
print("adv: ", len(adv_image_all))
test = np.concatenate([X_test, adv_image_all], axis=0)
true_test = np.concatenate([Y_test, adv_label_all], axis=0)
else:
test = X_test
true_test = Y_test
train = X_train
model = load_model(model_path)
pred_test_prob = model.predict(test)
pred_test = np.argmax(pred_test_prob, axis=1)
input = model.layers[0].output
lst = []
for index, layer in enumerate(model.layers):
if 'activation' in layer.name:
lst.append(index)
lst.append(len(model.layers) - 1)
# 是否deepxplore
if not deepxplore:
if model_name == model_conf.LeNet5: # 选择模型
layers = [
model.layers[2].output, model.layers[3].output,
model.layers[5].output, model.layers[6].output,
model.layers[8].output, model.layers[9].output,
model.layers[10].output
]
layers = list(zip(4 * ['conv'] + 3 * ['dense'], layers))
elif model_name == model_conf.LeNet1: # LeNet1
layers = [
model.layers[2].output,
model.layers[3].output,
model.layers[5].output,
model.layers[6].output,
model.layers[8].output,
]
# print(model.layers[1], model.layers[3], model.layers[6])
layers = list(zip(4 * ['conv'] + 1 * ['dense'], layers)) #
else: # ResNet20
layers = []
for index in lst:
layers.append(model.layers[index].output)
layers = list(zip(19 * ['conv'] + 1 * ['dense'], layers))
else:
if model_name == model_conf.LeNet5:
layers = [
model.layers[1].output, model.layers[3].output,
model.layers[4].output, model.layers[8].output,
model.layers[8].output, model.layers[9].output,
model.layers[10].output
]
layers = list(zip(4 * ['conv'] + 3 * ['dense'], layers))
elif model_name == model_conf.LeNet1: # LeNet1
layers = [
model.layers[1].output,
model.layers[3].output,
model.layers[4].output,
model.layers[6].output,
model.layers[8].output,
]
layers = list(zip(4 * ['conv'] + 1 * ['dense'], layers))
else:
layers = []
for index in lst:
if index != len(model.layers) - 1:
layers.append(model.layers[index - 1].output)
else:
layers.append(model.layers[index].output)
layers = list(zip(19 * ['conv'] + 1 * ['dense'], layers))
return input, layers, test, train, pred_test, true_test, pred_test_prob
def gen_data_cifar(model_name, use_adv=True, deepxplore=False):
model_path = model_conf.get_model_path(model_conf.cifar10, model_name)
(X_train, Y_train), (X_test, Y_test) = cifar10.load_data() # 32*32
X_train = X_train.astype('float32').reshape(-1, 32, 32, 3)
X_test = X_test.astype('float32').reshape(-1, 32, 32, 3)
X_train /= 255
X_test /= 255
Y_train = Y_train.reshape(-1)
Y_test = Y_test.reshape(-1)
# model_path = './model/model_cifar10.h5'
if use_adv:
attack_lst = ['cw', 'fgsm', 'jsma', 'bim']
adv_image_all = []
adv_label_all = []
for attack in attack_lst:
im, lab = model_conf.get_adv_path(attack, model_conf.cifar10,
model_name)
adv_image_all.append(np.load(im))
adv_label_all.append(np.load(lab))
adv_image_all = np.concatenate(adv_image_all, axis=0)
adv_label_all = np.concatenate(adv_label_all, axis=0)
test = np.concatenate([X_test, adv_image_all], axis=0)
true_test = np.concatenate([Y_test, adv_label_all], axis=0)
else:
test = X_test
true_test = Y_test
train = X_train
model = load_model(model_path)
pred_test_prob = model.predict(test)
pred_test = np.argmax(pred_test_prob, axis=1)
input = model.layers[0].output
lst = []
for index, layer in enumerate(model.layers):
if 'activation' in layer.name:
lst.append(index)
lst.append(len(model.layers) - 1)
if not deepxplore:
if model_name == model_conf.resNet20:
layers = []
for index in lst:
layers.append(model.layers[index].output)
layers = list(zip(19 * ['conv'] + 1 * ['dense'], layers))
else: # Vgg16
layers = []
for i in range(1, 19):
layers.append(model.layers[i].output)
for i in range(20, 23):
layers.append(model.layers[i].output)
layers = list(zip(18 * ['conv'] + 3 * ['dense'], layers))
else:
if model_name == model_conf.resNet20:
layers = []
for index in lst:
if index != len(model.layers) - 1:
layers.append(model.layers[index - 1].output)
else:
layers.append(model.layers[index].output)
layers = list(zip(19 * ['conv'] + 1 * ['dense'], layers))
else:
layers = []
for i in range(1, 19):
layers.append(model.layers[i].output)
for i in range(20, 23):
layers.append(model.layers[i].output)
layers = list(zip(18 * ['conv'] + 3 * ['dense'], layers))
return input, layers, test, train, pred_test, true_test, pred_test_prob, Y_train
def gen_data_svhn(model_name, use_adv=True, deepxplore=False):
model_path = model_conf.get_model_path(model_conf.svhn, model_name) #
(X_train, Y_train), (X_test, Y_test) = SVNH_DatasetUtil.load_data()
Y_train = np.argmax(Y_train, axis=1) # modify 原本是一堆矩阵,[0,0,0,1,0]代表第四类
Y_test = np.argmax(Y_test, axis=1)
if use_adv:
attack_lst = ['cw', 'fgsm', 'jsma', 'bim']
adv_image_all = []
adv_label_all = []
for attack in attack_lst:
im, lab = model_conf.get_adv_path(attack, model_conf.svhn,
model_name)
adv_image_all.append(np.load(im))
adv_label_all.append(np.load(lab))
adv_image_all = np.concatenate(adv_image_all, axis=0)
adv_label_all = np.concatenate(adv_label_all, axis=0)
test = np.concatenate([X_test, adv_image_all], axis=0)
true_test = np.concatenate([Y_test, adv_label_all], axis=0)
else:
test = X_test
true_test = Y_test
train = X_train
model = load_model(model_path)
pred_test_prob = model.predict(test)
pred_test = np.argmax(pred_test_prob, axis=1)
input = model.layers[0].output
lst = []
for index, layer in enumerate(model.layers):
if 'activation' in layer.name:
lst.append(index)
lst.append(len(model.layers) - 1)
if not deepxplore:
if model_name == model_conf.LeNet5:
layers = [
model.layers[2].output, model.layers[3].output,
model.layers[5].output, model.layers[6].output,
model.layers[8].output, model.layers[9].output,
model.layers[10].output
]
layers = list(zip(4 * ['conv'] + 3 * ['dense'], layers))
else: # Vgg16
layers = []
for i in range(1, 19):
layers.append(model.layers[i].output)
for i in range(20, 23):
layers.append(model.layers[i].output)
layers = list(zip(18 * ['conv'] + 3 * ['dense'], layers))
else:
if model_name == model_conf.LeNet5:
layers = [
model.layers[1].output, model.layers[3].output,
model.layers[4].output, model.layers[8].output,
model.layers[8].output, model.layers[9].output,
model.layers[10].output
]
layers = list(zip(4 * ['conv'] + 3 * ['dense'], layers))
else:
layers = []
for i in range(1, 19):
layers.append(model.layers[i].output)
for i in range(20, 23):
layers.append(model.layers[i].output)
layers = list(zip(18 * ['conv'] + 3 * ['dense'], layers))
return input, layers, test, train, pred_test, true_test, pred_test_prob, Y_train
def gen_data_mnist(
model_name,
use_adv=False,
deepxplore=False,
):
model_path = model_conf.get_model_path(model_conf.mnist, model_name) #
(X_train, Y_train), (X_test, Y_test) = mnist.load_data() # 28*28
X_train = X_train.astype('float32').reshape(-1, 28, 28, 1)
X_test = X_test.astype('float32').reshape(-1, 28, 28, 1)
X_train /= 255
X_test /= 255
# model_path = './model/model_mnist.hdf5'
if use_adv:
attack_lst = ['fgsm', 'jsma', 'bim', 'cw']
adv_image_all = []
adv_label_all = []
for attack in attack_lst:
im, lab = model_conf.get_adv_path(attack, model_conf.mnist,
model_name)
adv_image_all.append(np.load(im))
adv_label_all.append(np.load(lab))
adv_image_all = np.concatenate(adv_image_all, axis=0)
adv_label_all = np.concatenate(adv_label_all, axis=0)
test = np.concatenate([X_test, adv_image_all], axis=0)
true_test = np.concatenate([Y_test, adv_label_all], axis=0)
else:
test = X_test
true_test = Y_test
train = X_train
model = load_model(model_path)
pred_test_prob = model.predict(test)
pred_test = np.argmax(pred_test_prob, axis=1)
input = model.layers[0].output
if not deepxplore: #
if model_name == model_conf.LeNet5:
layers = [
model.layers[2].output, model.layers[3].output,
model.layers[5].output, model.layers[6].output,
model.layers[8].output, model.layers[9].output,
model.layers[10].output
]
layers = list(zip(4 * ['conv'] + 3 * ['dense'],
layers)) # 激活,池化,全连接
else:
layers = [
model.layers[2].output,
model.layers[3].output,
model.layers[5].output,
model.layers[6].output,
model.layers[8].output,
]
# print(model.layers[1], model.layers[3], model.layers[6])
layers = list(zip(4 * ['conv'] + 1 * ['dense'], layers)) #
else:
if model_name == model_conf.LeNet5:
layers = [
model.layers[1].output, model.layers[3].output,
model.layers[4].output, model.layers[8].output,
model.layers[8].output, model.layers[9].output,
model.layers[10].output
]
layers = list(zip(4 * ['conv'] + 3 * ['dense'],
layers)) # 卷积,池化,全连接
else:
layers = [
model.layers[1].output,
model.layers[3].output,
model.layers[4].output,
model.layers[6].output,
model.layers[8].output,
]
layers = list(zip(4 * ['conv'] + 1 * ['dense'], layers))
return input, layers, test, train, pred_test, true_test, pred_test_prob, Y_train
), label)
if __name__ == '__main__':
'''
mnist svhn fashion cifar10 cifar20
cw fgsm bim jsma
'''
data_lst = [
'fashion',
]
attack_lst = ['jsma', "fgsm", "cw", "bim"]
for dataset, attack in (itertools.product(data_lst, attack_lst)):
if dataset == "mnist":
model_path = model_conf.get_model_path(model_conf.mnist,
model_conf.LeNet1)
s = model_conf.mnist + model_conf.LeNet1
generate_adv_sample(s, model_path, dataset, attack)
model_path = model_conf.get_model_path(model_conf.mnist,
model_conf.LeNet5)
s = model_conf.mnist + model_conf.LeNet1
generate_adv_sample(s, model_path, dataset, attack)
elif dataset == "fashion":
model_path = model_conf.get_model_path(model_conf.fashion,
model_conf.LeNet1)
s = model_conf.fashion + model_conf.LeNet1
generate_adv_sample(s, model_path, dataset, attack)
model_path = model_conf.get_model_path(model_conf.fashion,
model_conf.resNet20)
s = model_conf.fashion + model_conf.resNet20
generate_adv_sample(s, model_path, dataset, attack)
def exec(dataset_name, model_name, deep_metric, **kwargs):
path = "./final_exp/model/{}/{}/model_mnist_ts_{}_{}.hdf5" # 模型储存路径
model_path = model_conf.get_model_path(dataset_name, model_name)
# 加载数据集
X_train, X_test, Y_train, Y_test = gen_data(dataset_name, model_name)
print('Train:{},Test:{}'.format(len(X_train), len(X_test)))
print(deep_metric, kwargs)
if len(kwargs) != 0:
params = "_" + "_".join(
[str(k) + "_" + str(v) for k, v in kwargs.items()])
else:
params = ""
ascending = True
# 获取阈值
threshold_arr = get_ts(len(X_test)) # 根据测试用例个数获取阈值
ts_percent_arr = get_ts()
# #进行试验
# print("params are ", kwargs)
#
for ixx in range(2): # 对于cam 和 ctm
pre_model_path = model_path # 初始化模型路径
cur_acc_arr = [] # 初始化结果数组
for i, ts in enumerate(threshold_arr): # 对于所有的阈值
ts = int(ts)
pre_model = load_model(pre_model_path) # 每次都重新加载上一次的模型
if ts == 0:
pre_acc = pre_model.evaluate(X_test, Y_test,
verbose=0)[1] # 初始化精度
cur_acc_arr.append(pre_acc) # 添加初始精度
print("origin acc is {}".format(pre_acc))
continue
# 做实验
print("exp: {} .....".format(i))
df = exp(pre_model,
dataset_name,
model_name,
X_train,
Y_train,
X_test,
Y_test,
deep_metric,
load_exist_table=load_exist_table,
**kwargs)
print("exp over...")
# 数据处理
df_case_rank_dict = df_process(df, ascending=ascending)
print("{} has {} rank method: {}".format(
deep_metric, len(df_case_rank_dict.keys()),
df_case_rank_dict.keys()))
if ixx == 0 and "cam" in df_case_rank_dict.keys():
key = "cam"
df_case_rank = df_case_rank_dict["cam"]
elif ixx == 1 and "ctm" in df_case_rank_dict.keys():
key = "ctm"
df_case_rank = df_case_rank_dict["ctm"]
elif ixx == 0 and "random" in df_case_rank_dict.keys():
key = "random"
df_case_rank = df_case_rank_dict["random"]
else:
break
# if deep_metric == "dsc" and key == "cam":
# continue
# if deep_metric == "lsc" and key == "cam":
# continue
if deep_metric == "deepgini2": # 采样获得用例的index
kernel = stats.gaussian_kde(df["score"])
val = kernel.evaluate(df["score"]) # 获得函数
pdf = val / sum(val) # 进行归一化
np.random.seed(42)
case_index = np.random.choice(df.index, p=pdf,
size=ts) # 采样获得用例
elif deep_metric == "deepgini3": # 采样获得用例的index
kernel = stats.gaussian_kde(df["score"])
val = kernel.evaluate(df["score"]) # 获得函数
val2 = val * df["score"].values # 乘以权重
pdf = val2 / sum(val2) # 进行归一化
np.random.seed(42)
case_index = np.random.choice(df.index, p=pdf,
size=ts) # 采样获得用例
print(
"===================init cases over=====================")
else:
case_index = df_case_rank[:ts]["case_index"] # 按阈值筛选用例
add_test_X = X_test[case_index]
add_test_Y = Y_test[case_index]
if only_add:
X_train_now = add_test_X
Y_train_now = add_test_Y
else:
X_train_now = np.r_[X_train, add_test_X]
Y_train_now = np.r_[Y_train, add_test_Y]
print('{}, 添加了{}个用例,现在的训练集长度为: {}'.format(i, ts, len(X_train_now)))
# path = "./final_exp/model/{}/{}/model_mnist_ts_{}_{}.hdf5" # 模型储存路径
path = path.format(dataset_name, model_name, ts_percent_arr[i],
deep_metric + params + "_" + key)
if load_exist_model and os.path.exists(path):
model = load_model(path)
cur_acc = model.evaluate(X_test, Y_test, verbose=0)[1]
else:
print("pre path is {} ,now path is {}".format(
pre_model_path, path))
cur_acc = model_fit(pre_model,
path,
X_train_now,
Y_train_now,
X_test,
Y_test,
batch_size=128,
name=dataset_name,
ts=ts_percent_arr[i],
verbose=1,
nb_epoch=10)[1]
print("pre_acc is {} , cur_acc is {} ,improve {}".format(
pre_acc, cur_acc, (cur_acc - pre_acc)))
cur_acc_arr.append(cur_acc)
pre_acc = cur_acc
pre_model_path = path
K.clear_session()
if len(cur_acc_arr) == len(ts_percent_arr): # 如果执行了cam/ctm,才记录结果
print(cur_acc_arr)
df_res = pd.DataFrame()
df_res["ts"] = ts_percent_arr
df_res["acc"] = cur_acc_arr
res_path = "./final_exp/res/{}/{}/{}_{}.csv".format(
dataset_name, model_name, deep_metric + params, key)
df_res.to_csv(res_path)