def set_random_seed(auto_var):
random_seed = auto_var.get_var("random_seed")
tf.set_random_seed(random_seed)
np.random.seed(random_seed)
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
keras.backend.set_session(sess)
keras.layers.core.K.set_learning_phase(0)
#tensorflow.keras.backend.set_session(sess)
sess.run(tf.global_variables_initializer())
auto_var.set_intermidiate_variable("sess", sess)
random_state = np.random.RandomState(auto_var.get_var("random_seed"))
auto_var.set_intermidiate_variable("random_state", random_state)
return random_state
def eps_accuracy(auto_var):
random_state = set_random_seed(auto_var)
ord = auto_var.get_var("ord")
dataset_name = auto_var.get_variable_name("dataset")
if 'fullmnist' in dataset_name or 'fullfashion' in dataset_name:
X, y, x_test, y_test, eps_list = auto_var.get_var("dataset")
idxs = np.arange(len(x_test))
random_state.shuffle(idxs)
tstX, tsty = x_test[idxs[:200]], y_test[idxs[:200]]
trnX, tstX = X.reshape((len(X), -1)), tstX.reshape((len(tstX), -1))
trny = y
else:
X, y, eps_list = auto_var.get_var("dataset")
idxs = np.arange(len(X))
random_state.shuffle(idxs)
trnX, tstX, trny, tsty = X[idxs[:-200]], X[idxs[-200:]], y[idxs[:-200]], y[idxs[-200:]]
scaler = MinMaxScaler()
trnX = scaler.fit_transform(trnX)
tstX = scaler.transform(tstX)
lbl_enc = OneHotEncoder(categories=[np.sort(np.unique(y))], sparse=False)
lbl_enc.fit(trny.reshape(-1, 1))
auto_var.set_intermidiate_variable("lbl_enc", lbl_enc)
ret = {}
results = []
auto_var.set_intermidiate_variable("trnX", trnX)
auto_var.set_intermidiate_variable("trny", trny)
model_name = auto_var.get_variable_name("model")
attack_name = auto_var.get_variable_name("attack")
if 'adv_rf' in model_name:
pre_model = auto_var.get_var_with_argument('model', model_name[4:])
pre_model.fit(trnX, trny)
if 'blackbox' in attack_name:
auto_var.set_intermidiate_variable("model", pre_model)
elif 'adv_nn' in model_name and 'blackbox' in attack_name:
pre_model = auto_var.get_var_with_argument('model', model_name[4:])
pre_model.fit(trnX, trny)
auto_var.set_intermidiate_variable("model", pre_model)
elif 'mlp' in model_name or 'logistic' in model_name:
auto_var.set_intermidiate_variable("eps_list", eps_list)
model = auto_var.get_var("model")
auto_var.set_intermidiate_variable("model", model)
model.fit(trnX, trny)
ret['trnX_len'] = len(trnX)
pred = model.predict(tstX)
ori_tstX, ori_tsty = tstX, tsty # len = 200
idxs = np.where(pred == tsty)[0]
random_state.shuffle(idxs)
tstX, tsty = tstX[idxs[:100]], tsty[idxs[:100]]
if len(tsty) != 100:
print("didn't got 100 testing examples, abort.")
ret['avg_pert'] = {'avg': 0, 'missed_count': 100,}
ret['tst_score'] = (model.predict(ori_tstX) == ori_tsty).mean()
if ('adv' in model_name) or ('advPruning' in model_name) or ('robustv2' in model_name):
ret['aug_len'] = len(model.augX)
return ret
#raise ValueError("didn't got 100 testing examples")
augX = None
if ('approxAP' in model_name) or ('adv' in model_name) or ('advPruning' in model_name) or ('robustv2' in model_name):
assert hasattr(model, 'augX')
auto_var.set_intermidiate_variable("trnX", model.augX)
auto_var.set_intermidiate_variable("trny", model.augy)
augX, augy = model.augX, model.augy
if augX is not None:
ret['aug_len'] = len(augX)
if len(tsty) != 100 or \
(np.unique(auto_var.get_intermidiate_variable('trny'))[0] != None and \
len(np.unique(auto_var.get_intermidiate_variable('trny'))) == 1):
tst_perturbs = np.array([np.zeros_like(tstX) for _ in range(len(eps_list))])
ret['single_label'] = True
attack_model = None
else:
attack_model = auto_var.get_var("attack")
tst_perturbs = attack_model.perturb(tstX, y=tsty, eps=eps_list)
ret['tst_score'] = (model.predict(ori_tstX) == ori_tsty).mean()
#########
if attack_model is not None and hasattr(attack_model, 'perts'):
perts = attack_model.perts
else:
perts = np.zeros_like(tstX)
for pert in tst_perturbs:
pred = model.predict(tstX + pert)
for i in range(len(pred)):
if (pred[i] != tsty[i]) and np.linalg.norm(perts[i])==0:
perts[i] = pert[i]
perts = perts.astype(float)
perts, missed_count = baseline_pert(model, trnX, tstX, tsty, perts, ord)
if len(np.unique(model.predict(trnX))) > 1:
assert (model.predict(tstX + perts) == tsty).sum() == 0, model.predict(tstX + perts) == tsty
else:
# ignore single label case
ret['single_label'] = True
ret['avg_pert'] = {
'avg': np.linalg.norm(perts, axis=1, ord=ord).mean().astype(float),
'missed_count': int(missed_count),
}
#########
results = estimate_model_roubstness(
model, tstX, tsty, tst_perturbs, eps_list, ord, with_baseline=False)
ret['results'] = results
baseline_results = estimate_model_roubstness(
model, tstX, tsty, tst_perturbs, eps_list, ord, with_baseline=True, trnX=trnX)
ret['baseline_results'] = baseline_results
print(json.dumps(auto_var.var_value))
print(json.dumps(ret))
return ret
def temp_fix(auto_var):
file_name = get_file_name(auto_var)
print(file_name)
if os.path.exists("%s.json" % file_name):
with open("%s.json" % file_name, "r") as f:
ret = json.load(f)
if "tst_score" in ret:
return
else:
return
random_state = set_random_seed(auto_var)
ord = auto_var.get_var("ord")
X, y, eps_list = auto_var.get_var("dataset")
idxs = np.arange(len(X))
random_state.shuffle(idxs)
trnX, tstX, trny, tsty = X[idxs[:-200]], X[idxs[-200:]], y[idxs[:-200]], y[
idxs[-200:]]
scaler = MinMaxScaler()
trnX = scaler.fit_transform(trnX)
tstX = scaler.transform(tstX)
lbl_enc = OneHotEncoder(categories=[np.sort(np.unique(y))], sparse=False)
#lbl_enc = OneHotEncoder(sparse=False)
lbl_enc.fit(trny.reshape(-1, 1))
auto_var.set_intermidiate_variable("lbl_enc", lbl_enc)
results = []
auto_var.set_intermidiate_variable("trnX", trnX)
auto_var.set_intermidiate_variable("trny", trny)
model_name = auto_var.get_variable_value("model")
attack_name = auto_var.get_variable_value("attack")
if 'adv_rf' in model_name:
pre_model = auto_var.get_var_with_argument('model', model_name[4:])
pre_model.fit(trnX, trny)
if 'blackbox' in attack_name:
auto_var.set_intermidiate_variable("model", pre_model)
elif 'adv_nn' in model_name and 'blackbox' in attack_name:
pre_model = auto_var.get_var_with_argument('model', model_name[4:])
pre_model.fit(trnX, trny)
auto_var.set_intermidiate_variable("model", pre_model)
model = auto_var.get_var("model")
auto_var.set_intermidiate_variable("model", model)
model.fit(trnX, trny)
pred = model.predict(tstX)
ori_tstX, ori_tsty = tstX, tsty # len = 200
idxs = np.where(pred == tsty)[0]
random_state.shuffle(idxs)
augX = None
if ('adv' in model_name) or ('advPruning'
in model_name) or ('robustv2' in model_name):
assert hasattr(model, 'augX')
auto_var.set_intermidiate_variable("trnX", model.augX)
auto_var.set_intermidiate_variable("trny", model.augy)
augX, augy = model.augX, model.augy
ret['tst_score'] = (model.predict(ori_tstX) == ori_tsty).mean()
with open("%s.json" % file_name, "w") as f:
json.dump(ret, f)
def eps_accuracy(auto_var):
random_state = set_random_seed(auto_var)
ord = auto_var.get_var("ord")
dataset_name = auto_var.get_variable_name("dataset")
if ('fullmnist' in dataset_name \
or 'fullfashion' in dataset_name \
or 'cifar' in dataset_name \
or 'fashion_mnist35f' in dataset_name \
or 'fashion_mnist06f' in dataset_name \
or 'mnist17f' in dataset_name \
or 'cifar' in dataset_name
):
X, y, x_test, y_test, eps_list = auto_var.get_var("dataset")
idxs = np.arange(len(x_test))
random_state.shuffle(idxs)
tstX, tsty = x_test[idxs[:200]], y_test[idxs[:200]]
idxs = np.arange(len(X))
random_state.shuffle(idxs)
X, y = X[idxs], y[idxs]
trnX, tstX = X.reshape((len(X), -1)), tstX.reshape((len(tstX), -1))
trny = y
else:
X, y, eps_list = auto_var.get_var("dataset")
idxs = np.arange(len(X))
random_state.shuffle(idxs)
trnX, tstX, trny, tsty = X[idxs[:-100]], X[idxs[-200:]], y[idxs[:-100]], y[idxs[-200:]]
scaler = MinMaxScaler()
trnX = scaler.fit_transform(trnX)
tstX = scaler.transform(tstX)
lbl_enc = OneHotEncoder(categories=[np.sort(np.unique(y))], sparse=False)
lbl_enc.fit(trny.reshape(-1, 1))
auto_var.set_intermidiate_variable("lbl_enc", lbl_enc)
ret = {}
results = []
x_adv = trnX;
y_adv = trny;
trnX = trnX[:-100];
trny = trny[:-100]
auto_var.set_intermidiate_variable("trnX", trnX)
auto_var.set_intermidiate_variable("trny", trny)
model_name = auto_var.get_variable_name("model")
attack_name = auto_var.get_variable_name("attack")
model = auto_var.get_var("model")
auto_var.set_intermidiate_variable("model", model)
model.fit(trnX, trny)
ret['trnX_len'] = len(trnX)
pred = model.predict(tstX)
baseline_acc = accuracy_score(pred, tsty)
# print(f"Baseline accuracy: {(pred == tsty).mean()}")
ori_tstX, ori_tsty = tstX, tsty # len = 200
pred_adv = model.predict(x_adv)
idxs = np.where(pred_adv == y_adv)[0]
random_state.shuffle(idxs)
x_adv, y_adv = x_adv[idxs[:len(trnX)]], y_adv[idxs[:len(trnX)]]
if len(x_adv) != len(trnX):
print("didn't got enough adversarial examples, abort.")
ret['avg_pert'] = {'avg': 0, 'missed_count': 100,}
ret['tst_score'] = (model.predict(ori_tstX) == ori_tsty).mean()
if ('adv' in model_name) or ('advPruning' in model_name) or ('robustv2' in model_name):
ret['aug_len'] = len(model.augX)
return ret
#raise ValueError("didn't got 100 testing examples")
# if len(tsty) != 100 or \
# (np.unique(auto_var.get_intermidiate_variable('trny'))[0] != None and \
# len(np.unique(auto_var.get_intermidiate_variable('trny'))) == 1):
# tst_perturbs = np.array([np.zeros_like(tstX) for _ in range(len(eps_list))])
# ret['single_label'] = True
# attack_model = None
# else:
# attack_model = auto_var.get_var("attack")
# adv_perturbs = attack_model.perturb(x_adv, y=y_adv, eps=eps_list)
attack_model = auto_var.get_var("attack")
adv_perturbs = attack_model.perturb(x_adv, y=y_adv, eps=eps_list)
ret['tst_score'] = (model.predict(ori_tstX) == ori_tsty).mean()
#########
# perts = attack_model.perts
if attack_model is not None and hasattr(attack_model, 'perts'):
perts = attack_model.perts
else:
perts = np.zeros_like(x_adv)
for pert in adv_perturbs:
pred = model.predict(x_adv + pert)
for i in range(len(pred)):
if (pred[i] != y_adv[i]) and np.linalg.norm(perts[i])==0:
perts[i] = pert[i]
perts = perts.astype(float)
# missed_count = verify_adv(model, x_adv, perts, y_adv)
print('Training size : ', len(trnX))
print('Baseline accuracy : %s' % '{0:.3%}'.format(baseline_acc))
# print('Failed to find : %s' % '{0:.3%}'.format(missed_count))
# noise_scale = auto_var.get_var("noise_scale")
knockoff_acc = knockoff(model, x_adv, y_adv, perts, 0.9, ori_tstX, ori_tsty)
print('Knockoff model accuracy : %s' % '{0:.3%}'.format(knockoff_acc))
# print('Perturbation : %s' % '{0:.3%}'.format(np.linalg.norm(perts, axis=1, ord=ord).mean().astype(float)))
perts, missed_count = baseline_pert(model, trnX, x_adv, y_adv, perts, ord)
print('Missed count : ', missed_count)
#if len(np.unique(model.predict(trnX))) > 1:
# assert (model.predict(tstX + perts) == tsty).sum() == 0, model.predict(tstX + perts) == tsty
#else:
# # ignore single label case
# ret['single_label'] = True
ret['avg_pert'] = {
'avg': np.linalg.norm(perts, axis=1, ord=ord).mean().astype(float),
'missed_count': int(missed_count),
}
print('Distortion avg: ', np.linalg.norm(perts, axis=1, ord=ord).mean().astype(float))
#########
# results = estimate_model_roubstness(
# model, tstX, tsty, tst_perturbs, eps_list, ord, with_baseline=False)
# ret['results'] = results
# baseline_results = estimate_model_roubstness(
# model, tstX, tsty, tst_perturbs, eps_list, ord, with_baseline=True, trnX=trnX)
# ret['baseline_results'] = baseline_results
# print(json.dumps(auto_var.var_value))
# print(json.dumps(ret))
return ret
