def detect(args):
assert args.dataset in DATASETS, \
"Dataset parameter must be either 'mnist', 'cifar' or 'svhn'"
assert args.attack in ATTACKS, \
"Train attack must be either 'fgsm', 'bim-a', 'bim-b', " \
"'jsma', 'cw-l2'"
assert args.test_attack in ATTACKS, \
"Test attack must be either 'fgsm', 'bim-a', 'bim-b', " \
"'jsma', 'cw-l2'"
characteristics = args.characteristics.split(',')
for char in characteristics:
assert char in CHARACTERISTICS, \
"Characteristic(s) to use 'kd', 'bu', 'lid'"
print("Loading train attack: %s" % args.attack)
X, Y = load_characteristics(args.dataset, args.attack, characteristics)
# standarization
scaler = MinMaxScaler().fit(X)
X = scaler.transform(X)
# X = scale(X) # Z-norm
# test attack is the same as training attack
X_train, Y_train, X_test, Y_test = block_split(X, Y)
if args.test_attack != args.attack:
# test attack is a different attack
print("Loading test attack: %s" % args.test_attack)
X_test, Y_test = load_characteristics(args.dataset, args.test_attack,
characteristics)
_, _, X_test, Y_test = block_split(X_test, Y_test)
# apply training normalizer
X_test = scaler.transform(X_test)
# X_test = scale(X_test) # Z-norm
print("Train data size: ", X_train.shape)
print("Test data size: ", X_test.shape)
## Build detector
print(
"LR Detector on [dataset: %s, train_attack: %s, test_attack: %s] with:"
% (args.dataset, args.attack, args.test_attack))
lr = train_lr(X_train, Y_train)
## Evaluate detector
y_pred = lr.predict_proba(X_test)[:, 1]
y_label_pred = lr.predict(X_test)
# AUC
_, _, auc_score = compute_roc(Y_test, y_pred, plot=False)
precision = precision_score(Y_test, y_label_pred)
recall = recall_score(Y_test, y_label_pred)
y_label_pred = lr.predict(X_test)
acc = accuracy_score(Y_test, y_label_pred)
print(
'Detector ROC-AUC score: %0.4f, accuracy: %.4f, precision: %.4f, recall: %.4f'
% (auc_score, acc, precision, recall))
return lr, auc_score, scaler
def detect(args):
assert args.dataset in DATASETS, \
"Dataset parameter must be either 'mnist', 'cifar' or 'svhn'"
# assert args.attack in ATTACKS, \
# "Training attack must be either 'fgsm', 'bim-a', 'bim-b', " \
# "'jsma', 'cw-l0, cw-l2 or cw-li'"
# if args.test_attack is not None:
# assert args.test_attack in ATTACKS, \
# "Test attack must be either 'fgsm', 'bim-a', 'bim-b', " \
# "'jsma', 'cw-l0, cw-l2 or cw-li'"
artifacts = args.artifacts.split(',')
for atf in artifacts:
assert atf in ARTIFACTS, \
"Artifact(s) to use 'kd', 'bu', 'lid'"
X, Y = load_artifacts(args.dataset, args.attack, artifacts)
#X = X[:, :-1]
# normalization
scaler = MinMaxScaler().fit(X)
X = scaler.transform(X)
# X = scale(X) # Z-norm
if args.test_attack is None or args.test_attack == args.attack:
# test attack is the same as training attack
X_train, Y_train, X_test, Y_test = block_split(X, Y)
else:
# test attack is a different attack
# load the test attack for testing
print("Loading test attack: %s" % args.test_attack)
X_train, Y_train = X, Y
X_test, Y_test = load_artifacts(args.dataset, args.test_attack,
artifacts)
# using training normalizer
X_test = scaler.transform(X_test)
# X_test = scale(X_test) # Z-norm
# for adversarial testing - only use adversarial examples:
num_samples = X_test.shape[0]
partition = int(num_samples / 3)
# X_test, Y_test = X_test[:partition], Y_test[:partition] # only test the accuracy on the advs
X_test, Y_test = X_test[:2 *
partition], Y_test[:2 *
partition] # test accuracy on normal and advs
# X_noisy, Y_noisy = X_test[2 * partition:], Y_test[2*partition:]
# test -- use one layer of lid
# X_train = X_train[:, -1, None]
# X_test = X_test[:, -1, None]
print("Train samples size: ", X_train.shape)
print("Test samples size: ", X_test.shape)
# print(X_train[0])
# print(X_test[0])
## Build detector
print(
"LR Detector on [dataset: %s, train_attack: %s, test_attack: %s] with:"
% (args.dataset, args.attack, args.test_attack))
lr = train_lr(X_train, Y_train)
## Evaluate detector
# Compute logistic regression model predictions
y_pred = lr.predict_proba(X_test)[:, 1]
# Compute AUC
n_samples = len(X_test)
_, _, auc_score = compute_roc(Y_test, y_pred)
y_label_pred = lr.predict(X_test)
acc = accuracy_score(Y_test, y_label_pred)
print('Detector ROC-AUC score: %0.4f, accuracy: %.4f' % (auc_score, acc))
return lr, auc_score, scaler