def exp(coverage, use_adv, std=0, k=1, k_bins=1000):
input, layers, test, train, pred_test, true_test, pred_test_prob = gen_data(
use_adv)
rank_lst2 = None
if coverage == 'kmnc':
# 只能贪心排
km = metrics.kmnc(train, input, layers, k_bins=k_bins)
rank_lst = km.rank_fast(test)
rate = km.fit(test)
elif coverage == 'nbc':
#可以贪心排
#可以单个样本比较排
#0 0.5 1
bc = metrics.nbc(train, input, layers, std=std)
rank_lst = bc.rank_fast(test, use_lower=True)
rank_lst2 = bc.rank_2(test, use_lower=True)
rate = bc.fit(test, use_lower=True)
elif coverage == 'snac':
#可以贪心排
#可以单个样本比较排
#0 0.5 1
bc = metrics.nbc(train, input, layers, std=std)
rank_lst = bc.rank_fast(test, use_lower=False)
rank_lst2 = bc.rank_2(test, use_lower=False)
rate = bc.fit(test, use_lower=False)
elif coverage == 'tknc':
# 只能贪心排
#1 2 3
tk = metrics.tknc(test, input, layers, k=k)
rank_lst = tk.rank(test)
rate = tk.fit(list(range(len(test))))
df = pd.DataFrame([])
df['right'] = (pred_test == true_test).astype('int')
df['cam'] = 0
df['cam'].loc[rank_lst] = list(range(1, len(rank_lst) + 1))
if rank_lst2 is not None:
df['ctm'] = 0
df['ctm'].loc[rank_lst2] = list(range(1, len(rank_lst2) + 1))
if use_adv:
dataset = 'mnist_adv'
else:
dataset = 'mnist'
df['rate'] = rate
if coverage == 'kmnc':
pass
#df.to_csv('./output_mnist/{}_{}_k_bins_{}.csv'.format(dataset,coverage,k_bins))
elif coverage == 'nbc':
pass
#df.to_csv('./output_mnist/{}_{}_std_{}.csv'.format(dataset,coverage,std))
elif coverage == 'snac':
pass
#df.to_csv('./output_mnist/{}_{}_std_{}.csv'.format(dataset,coverage,std))
elif coverage == 'tknc':
pass
def exp(model_name, coverage, use_adv, std=0.0, k=1, k_bins=1000):
input, layers, test, train, pred_test, true_test, pred_test_prob = gen_data(
model_name, use_adv=use_adv)
rank_lst2 = None
rank_lst_time = None
rank_lst2_time = None
if coverage == 'kmnc':
# 只能贪心排
km = metrics.kmnc(train, input, layers, k_bins=k_bins)
start = time.time()
rank_lst = km.rank_fast(test)
end = time.time()
rank_lst_time = start - end
# rate = km.fit(test)
if rank_lst is not None:
rate = km.fit(test)
elif coverage == 'nbc':
# 可以贪心排
# 可以单个样本比较排
# 0 0.5 1
bc = metrics.nbc(train, input, layers, std=std)
start = time.time()
rank_lst = bc.rank_fast(test, use_lower=True)
end = time.time()
rank_lst_time = start - end
start = time.time()
rank_lst2 = bc.rank_2(test, use_lower=True)
end = time.time()
rank_lst2_time = start - end
rate = bc.fit(test, use_lower=True)
elif coverage == 'snac':
# 可以贪心排
# 可以单个样本比较排
# 0 0.5 1
bc = metrics.nbc(train, input, layers, std=std)
start = time.time()
rank_lst = bc.rank_fast(test, use_lower=False)
end = time.time()
rank_lst_time = start - end
start = time.time()
rank_lst2 = bc.rank_2(test, use_lower=False)
end = time.time()
rank_lst2_time = start - end
rate = bc.fit(test, use_lower=False)
elif coverage == 'tknc':
# 只能贪心排
# 1 2 3
start = time.time()
tk = metrics.tknc(test, input, layers, k=k)
rank_lst = tk.rank(test)
end = time.time()
rank_lst_time = start - end
rate = tk.fit(list(range(len(test))))
df = pd.DataFrame([])
df['right'] = (pred_test == true_test).astype('int')
df['cam'] = 0
df['ctm'] = 0
df['cam'].loc[rank_lst] = list(range(1, len(rank_lst) + 1))
df['cam_time'] = rank_lst_time
df['ctm_time'] = rank_lst2_time
if rank_lst2 is not None:
df['ctm'].loc[rank_lst2] = list(range(1, len(rank_lst2) + 1))
if use_adv:
dataset = 'fashion_adv'
else:
dataset = 'fashion'
df['rate'] = rate
if coverage == 'kmnc':
pass
df.to_csv('./all_output/output_fashion/{}/{}_{}_k_bins_{}.csv'.format(
model_name, dataset, coverage, k_bins))
elif coverage == 'nbc':
pass
df.to_csv('./all_output/output_fashion/{}/{}_{}_std_{}.csv'.format(
model_name, dataset, coverage, std))
elif coverage == 'snac':
pass
df.to_csv('./all_output/output_fashion/{}/{}_{}_std_{}.csv'.format(
model_name, dataset, coverage, std))
elif coverage == 'tknc':
pass
df.to_csv('./all_output/output_fashion/{}/{}_{}_k_{}.csv'.format(
model_name, dataset, coverage, k))
def exp(model,
name,
model_name,
X_train,
Y_train,
X_test,
Y_test,
deep_metric,
load_exist_table=True,
**kwargs):
pred_test_prob = model.predict(X_test)
pred_test = np.argmax(pred_test_prob, axis=1)
Y_test = np.argmax(Y_test, axis=1)
rank_lst2 = None
rank_lst = None
rank_lst_time = None
rank_lst2_time = None
rate = 0
if len(kwargs) != 0:
params = "_" + "_".join(
[str(k) + "_" + str(v) for k, v in kwargs.items()])
else:
params = ""
path = './final_exp/res/{}/{}_{}_{}.csv'.format(name, name, model_name,
deep_metric + params)
if load_exist_table:
# 如果有文件就不要在做实验了
if os.path.exists(path):
print("load_exist_table")
df = pd.read_csv(path, index_col=0)
return df
df = pd.DataFrame([])
if deep_metric == "deepgini":
rank_lst = metrics.deep_metric(pred_test_prob)
elif deep_metric == "deepgini2" or deep_metric == "deepgini3":
rank_lst, score = metrics.deep_metric2(pred_test_prob)
df["score"] = score
else:
if deep_metric == "nac":
input, layers, = get_layers(model,
name,
model_name,
deepxplore=True)
else:
input, layers, = get_layers(model,
name,
model_name,
deepxplore=False)
if deep_metric == "nac":
ac = metrics.nac(X_test, input, layers, t=kwargs["t"])
rate = ac.fit()
start = time.time()
rank_lst = ac.rank_fast(X_test)
end = time.time()
rank_lst_time = start - end
start = time.time()
rank_lst2 = ac.rank_2(X_test)
end = time.time()
rank_lst2_time = start - end
if deep_metric == 'kmnc':
# 只能贪心排
km = metrics.kmnc(X_train, input, layers, k_bins=kwargs["k"])
start = time.time()
rank_lst = km.rank_fast(X_test)
end = time.time()
rank_lst_time = start - end
rate = km.fit(X_test)
elif deep_metric == 'nbc':
# 可以贪心排
# 可以单个样本比较排
# 0 0.5 1
bc = metrics.nbc(X_train, input, layers, std=kwargs["std"])
start = time.time()
rank_lst = bc.rank_fast(X_test, use_lower=True)
end = time.time()
rank_lst_time = start - end
start = time.time()
rank_lst2 = bc.rank_2(X_test, use_lower=True)
end = time.time()
rank_lst2_time = start - end
rate = bc.fit(X_test, use_lower=True)
elif deep_metric == 'snac':
# 可以贪心排
# 可以单个样本比较排
# 0 0.5 1
bc = metrics.nbc(X_train, input, layers, std=kwargs["std"])
start = time.time()
rank_lst = bc.rank_fast(X_test, use_lower=False)
end = time.time()
rank_lst_time = start - end
start = time.time()
rank_lst2 = bc.rank_2(X_test, use_lower=False)
end = time.time()
rank_lst2_time = start - end
rate = bc.fit(X_test, use_lower=False)
elif deep_metric == 'tknc':
# 只能贪心排
# 1 2 3
tk = metrics.tknc(X_test, input, layers, k=kwargs["k"])
start = time.time()
rank_lst = tk.rank(X_test)
end = time.time()
rank_lst_time = start - end
rate = tk.fit(list(range(len(X_test))))
elif deep_metric == 'lsc':
label = np.argmax(Y_train, axis=1)
st = time.time() # 计算排序时间
model = metrics.LSC(X_train,
label,
input, [layers[kwargs["index"]]],
u=100)
rate = model.fit(X_test, pred_test)
en = time.time()
pre_time = st - en
start = time.time()
rank_lst2 = model.rank_2()
end = time.time()
rank_lst2_time = start - end + pre_time
start = time.time()
rank_lst = model.rank_fast()
end = time.time()
rank_lst_time = start - end + pre_time
elif deep_metric == "dsc":
label = np.argmax(Y_train, axis=1)
st = time.time() # 计算排序时间
model = metrics.DSC(X_train, label, input, layers)
rate = model.fit(X_test, pred_test)
en = time.time()
pre_time = st - en
start = time.time()
rank_lst2 = model.rank_2()
end = time.time()
rank_lst2_time = start - end + pre_time
start = time.time()
rank_lst = model.rank_fast()
end = time.time()
rank_lst_time = start - end + pre_time
df['right'] = (pred_test == Y_test).astype('int')
df['cam'] = 0
df['ctm'] = 0
df['cam_time'] = rank_lst_time
df['ctm_time'] = rank_lst2_time
if rank_lst is not None:
df['cam'].loc[rank_lst] = list(range(1, len(rank_lst) + 1))
if rank_lst2 is not None:
df['ctm'].loc[rank_lst2] = list(range(1, len(rank_lst2) + 1))
df['rate'] = rate
if deep_metric == "random":
df = df.sample(frac=1, random_state=41).reset_index()
df.to_csv(path)
return df