def detect(args):
    assert args.dataset in DATASETS, \
        "Dataset parameter must be either 'mnist', 'cifar' or 'svhn'"
    assert args.attack in ATTACKS, \
        "Train attack must be either 'fgsm', 'bim-a', 'bim-b', " \
        "'jsma', 'cw-l2'"
    assert args.test_attack in ATTACKS, \
        "Test attack must be either 'fgsm', 'bim-a', 'bim-b', " \
        "'jsma', 'cw-l2'"
    characteristics = args.characteristics.split(',')
    for char in characteristics:
        assert char in CHARACTERISTICS, \
            "Characteristic(s) to use 'kd', 'bu', 'lid'"

    print("Loading train attack: %s" % args.attack)
    X, Y = load_characteristics(args.dataset, args.attack, characteristics)

    # standarization
    scaler = MinMaxScaler().fit(X)
    X = scaler.transform(X)
    # X = scale(X) # Z-norm

    # test attack is the same as training attack
    X_train, Y_train, X_test, Y_test = block_split(X, Y)
    if args.test_attack != args.attack:
        # test attack is a different attack
        print("Loading test attack: %s" % args.test_attack)
        X_test, Y_test = load_characteristics(args.dataset, args.test_attack,
                                              characteristics)
        _, _, X_test, Y_test = block_split(X_test, Y_test)

        # apply training normalizer
        X_test = scaler.transform(X_test)
        # X_test = scale(X_test) # Z-norm

    print("Train data size: ", X_train.shape)
    print("Test data size: ", X_test.shape)

    ## Build detector
    print(
        "LR Detector on [dataset: %s, train_attack: %s, test_attack: %s] with:"
        % (args.dataset, args.attack, args.test_attack))
    lr = train_lr(X_train, Y_train)

    ## Evaluate detector
    y_pred = lr.predict_proba(X_test)[:, 1]
    y_label_pred = lr.predict(X_test)

    # AUC
    _, _, auc_score = compute_roc(Y_test, y_pred, plot=False)
    precision = precision_score(Y_test, y_label_pred)
    recall = recall_score(Y_test, y_label_pred)

    y_label_pred = lr.predict(X_test)
    acc = accuracy_score(Y_test, y_label_pred)
    print(
        'Detector ROC-AUC score: %0.4f, accuracy: %.4f, precision: %.4f, recall: %.4f'
        % (auc_score, acc, precision, recall))

    return lr, auc_score, scaler
コード例 #2
0
def detect(args):
    assert args.dataset in DATASETS, \
        "Dataset parameter must be either 'mnist', 'cifar' or 'svhn'"
    # assert args.attack in ATTACKS, \
    #     "Training attack must be either 'fgsm', 'bim-a', 'bim-b', " \
    #     "'jsma', 'cw-l0, cw-l2 or cw-li'"
    # if args.test_attack is not None:
    #     assert args.test_attack in ATTACKS, \
    #         "Test attack must be either 'fgsm', 'bim-a', 'bim-b', " \
    #         "'jsma', 'cw-l0, cw-l2 or cw-li'"
    artifacts = args.artifacts.split(',')
    for atf in artifacts:
        assert atf in ARTIFACTS, \
            "Artifact(s) to use 'kd', 'bu', 'lid'"

    X, Y = load_artifacts(args.dataset, args.attack, artifacts)
    #X = X[:, :-1]

    # normalization
    scaler = MinMaxScaler().fit(X)
    X = scaler.transform(X)
    # X = scale(X) # Z-norm

    if args.test_attack is None or args.test_attack == args.attack:
        # test attack is the same as training attack
        X_train, Y_train, X_test, Y_test = block_split(X, Y)
    else:
        # test attack is a different attack
        # load the test attack for testing
        print("Loading test attack: %s" % args.test_attack)
        X_train, Y_train = X, Y
        X_test, Y_test = load_artifacts(args.dataset, args.test_attack,
                                        artifacts)
        # using training normalizer
        X_test = scaler.transform(X_test)
        # X_test = scale(X_test) # Z-norm

        # for adversarial testing - only use adversarial examples:
        num_samples = X_test.shape[0]
        partition = int(num_samples / 3)
        # X_test, Y_test = X_test[:partition], Y_test[:partition] # only test the accuracy on the advs
        X_test, Y_test = X_test[:2 *
                                partition], Y_test[:2 *
                                                   partition]  # test accuracy on normal and advs
        # X_noisy, Y_noisy = X_test[2 * partition:], Y_test[2*partition:]

    # test -- use one layer of lid
    # X_train = X_train[:, -1, None]
    # X_test = X_test[:, -1, None]

    print("Train samples size: ", X_train.shape)
    print("Test samples size: ", X_test.shape)

    # print(X_train[0])
    # print(X_test[0])

    ## Build detector
    print(
        "LR Detector on [dataset: %s, train_attack: %s, test_attack: %s] with:"
        % (args.dataset, args.attack, args.test_attack))
    lr = train_lr(X_train, Y_train)

    ## Evaluate detector
    # Compute logistic regression model predictions
    y_pred = lr.predict_proba(X_test)[:, 1]
    # Compute AUC
    n_samples = len(X_test)
    _, _, auc_score = compute_roc(Y_test, y_pred)

    y_label_pred = lr.predict(X_test)
    acc = accuracy_score(Y_test, y_label_pred)
    print('Detector ROC-AUC score: %0.4f, accuracy: %.4f' % (auc_score, acc))

    return lr, auc_score, scaler