def _retrain(self): # Retrain the model with poisoned data learning_model = svm.SVC(probability=True, kernel='linear') self.attack_learner = SimpleLearner(learning_model, self.attack_instances) self.attack_learner.train() self.attack_training_pred_labels = self.attack_learner.predict( self.training_instances) self.attack_testing_pred_labels = self.attack_learner.predict( self.testing_instances)
def _setup(self): if self.verbose: print('Training sample size: ', len(self.training_instances), '/400\n', sep='') # Setting the default learner learning_model = svm.SVC(probability=True, kernel='linear') self.learner = SimpleLearner(learning_model, self.training_instances) self.learner.train() self.training_pred_labels = self.learner.predict( self.training_instances) self.testing_pred_labels = self.learner.predict(self.testing_instances)
def test_label_flipping(): print('\n#################################################################') print('START label flipping attack.\n') begin = time.time() # Data processing unit # The path is an index of 400 testing samples(raw email data). dataset = EmailDataset(path='./data_reader/data/raw/trec05p-1/test-400', binary=True, raw=True) training_data = load_dataset(dataset) print('Training sample size: ', len(training_data), '/400\n', sep='') # Randomly cut dataset in approximately half rand_choices = np.random.binomial(1, 0.5, len(training_data)) new_training_data = [] predict_data = [] for i in range(len(training_data)): if rand_choices[i] == 1: new_training_data.append(training_data[i]) else: predict_data.append(training_data[i]) training_data = new_training_data # Setting the default learner # Test simple learner svm learning_model = svm.SVC(probability=True, kernel='linear') learner = SimpleLearner(learning_model, training_data) learner.train() orig_learner = deepcopy(learner) # Execute the attack cost = list(np.random.binomial(2, 0.5, len(training_data))) total_cost = 0.3 * len(training_data) # flip around ~30% of the labels attacker = LabelFlipping(learner, cost, total_cost, verbose=True) attack_data = attacker.attack(training_data) flip_vector = [] # 0 -> flipped, 1 -> not flipped for i in range(len(attack_data)): if attack_data[i].get_label() != training_data[i].get_label(): flip_vector.append(0) else: flip_vector.append(1) print('Flip vector with 0 -> flipped and 1 -> not flipped: \n', np.array(flip_vector), '\n') original_pred_labels = learner.predict(training_data) # Retrain the model with poisoned data learning_model = svm.SVC(probability=True, kernel='linear') learner = SimpleLearner(learning_model, attack_data) learner.train() ############################################################################ # Calculate statistics with training data attack_pred_labels = learner.predict(training_data) # predict w/ orig label (orig_precent_correct, attack_precent_correct, difference) = calculate_correct_percentages(original_pred_labels, attack_pred_labels, training_data) print('###################################################################') print('Predictions with training dataset:') print('Original correct percentage: ', orig_precent_correct, '%') print('Attack correct percentage: ', attack_precent_correct, '%') print('Difference: ', difference, '%') ############################################################################ # Calculate statistics with predict data (other half of dataset) original_pred_labels = orig_learner.predict(predict_data) attack_pred_labels = learner.predict(predict_data) (orig_precent_correct, attack_precent_correct, difference) = calculate_correct_percentages(original_pred_labels, attack_pred_labels, predict_data) print('###################################################################') print('Predictions with other half of dataset:') print('Original correct percentage: ', orig_precent_correct, '%') print('Attack correct percentage: ', attack_precent_correct, '%') print('Difference: ', difference, '%') end = time.time() print('\nTotal time: ', round(begin - end, 2), 's', '\n', sep='') print('\nEND label flipping attack.') print('#################################################################\n')
ls = [x for x, y in zip(x_test, y_pred) if x.label == 1 and y == 1] print("{0} malicious instances are being detected initially") return ls, [x.label for x in ls] dataset = EmailDataset(path='./data_reader/data/raw/trec05p-1/test-400', binary=True, raw=True) training_, testing_ = dataset.split({'train': 60, 'test': 40}) training_data = load_dataset(training_) testing_data = load_dataset(testing_) test_true_label = [x.label for x in testing_data] # test simple learner svm learning_model = svm.SVC(probability=True, kernel='linear') learner1 = SimpleLearner(learning_model, training_data) learner1.train() predictions = learner1.predict(testing_data) print("======== initial prediction =========") print(summary(predictions, test_true_label)) # Note: should not use only malicious data attacker = GoodWord(n=500) attacker.set_adversarial_params(learner1, testing_data) new_testing_data = attacker.attack(testing_data) predictions2 = learner1.predict(new_testing_data) print("========= post-attack prediction =========") print("post attack preds" + str(predictions2)) print(type(predictions2).__name__)
def NB_learner(data): learner_model = GaussianNB() learner = SimpleLearner(model=learner_model, training_instances=data['training_data']) learner.train() return learner
def simple_learner(data): learning_model = svm.SVC(probability=True, kernel='linear') learner = SimpleLearner(learning_model, data['training_data']) learner.train() return learner
def test_iterative_retraining_learner(): print() print( '###################################################################') print('START TRIM learner test.\n') begin = time.time() if len(sys.argv) == 2 and sys.argv[1] in [ 'label-flipping', 'k-insertion', 'data-modification' ]: attacker_name = sys.argv[1] else: attacker_name = 'label-flipping' # Data processing unit # The path is an index of 400 testing samples(raw email data). dataset = EmailDataset(path='./data_reader/data/raw/trec05p-1/test-400', binary=True, raw=True) training_data, testing_data = dataset.split({'train': 20, 'test': 80}) training_data = load_dataset(training_data) testing_data = load_dataset(testing_data) print('Training sample size: ', len(training_data), '/400\n', sep='') # Setting the default learner learning_model = svm.SVC(probability=True, kernel='linear') learner = SimpleLearner(learning_model, training_data) learner.train() original_pred_labels = learner.predict(training_data) orig_learner = deepcopy(learner) # Execute the attack if attacker_name == 'label-flipping': cost = list(np.random.binomial(2, 0.5, len(training_data))) total_cost = 0.3 * len(training_data) # flip around ~30% of the labels attacker = LabelFlipping(learner, cost, total_cost, verbose=True) elif attacker_name == 'k-insertion': number_to_add = int(0.25 * len(training_data)) attacker = KInsertion(learner, training_data[0], number_to_add=number_to_add, verbose=True) else: # attacker_name == 'data-modification' lnr = orig_learner.model.learner eye = np.eye(training_data[0].get_feature_count(), dtype=int) orig_theta = lnr.decision_function(eye) - lnr.intercept_[0] target_theta = deepcopy(orig_theta) spam_instances = [] for inst in training_data + testing_data: if inst.get_label() == 1: spam_instances.append(inst) spam_features, ham_features = get_spam_features(spam_instances) # Set features to recognize spam as ham for index in spam_features: target_theta[index] = -10 for index in ham_features: target_theta[index] = 0.01 print('Features selected: ', np.array(spam_features)) print('Number of features: ', len(spam_features)) attacker = DataModification(orig_learner, target_theta, verbose=True) print( '###################################################################') print('START', attacker_name, 'attack.\n') attack_data = attacker.attack(training_data) print('\nEND', attacker_name, 'attack.') print( '###################################################################') print() # Retrain the model with poisoned data learning_model = svm.SVC(probability=True, kernel='linear') learner = SimpleLearner(learning_model, attack_data) learner.train() print( '###################################################################') print('START Iterative Retraining learner.\n') ### print('\nEND Iterative Retraining learner.') print( '###################################################################') print() ############################################################################ # Calculate statistics with training data attack_pred_labels = learner.predict( training_data) # predict w/ orig label (orig_precent_correct, attack_precent_correct, difference) = calculate_correct_percentages(original_pred_labels, attack_pred_labels, training_data) print( '###################################################################') print('Predictions with training dataset:') print('Original correct percentage: ', orig_precent_correct, '%') print('Attack correct percentage: ', attack_precent_correct, '%') print('Difference: ', difference, '%') ############################################################################ # Calculate statistics with predict data (other half of dataset) original_pred_labels = orig_learner.predict(testing_data) attack_pred_labels = learner.predict(testing_data) (orig_precent_correct, attack_precent_correct, difference) = calculate_correct_percentages(original_pred_labels, attack_pred_labels, testing_data) print( '###################################################################') print('Predictions with other half of dataset:') print('Original correct percentage: ', orig_precent_correct, '%') print('Attack correct percentage: ', attack_precent_correct, '%') print('Difference: ', difference, '%') ############################################################################ # Calculate statistics with trim learner data = training_data + testing_data trim_pred_labels = trim_learner.predict(data) normal_pred_labels = learner.predict(data) (trim_percent_correct, normal_percent_correct, difference) = calculate_correct_percentages(trim_pred_labels, normal_pred_labels, data) print( '###################################################################') print('Predictions using TRIM learner:') print('TRIM learner percentage: ', trim_percent_correct, '%') print('Simple learner correct percentage: ', normal_percent_correct, '%') print('Difference: ', difference, '%') end = time.time() print('\nTotal time: ', round(begin - end, 2), 's', '\n', sep='') print('\nEND TRIM learner test.') print( '###################################################################') print()
def test_k_insertion(): """ Use as follows: python3 adlib/tests/adversaries/k_insertion_test.py #-TO-ADD #-ITERATIONS """ print( '\n#################################################################') print('START k-insertion attack.\n') if len(sys.argv) > 2: number_to_add = int(sys.argv[1]) num_iterations = int(sys.argv[2]) else: number_to_add = 1 num_iterations = 4 # Data processing unit # The path is an index of 400 testing samples(raw email data). dataset = EmailDataset(path='./data_reader/data/raw/trec05p-1/test-400', binary=True, raw=True) training_data = load_dataset(dataset) # Randomly choose ~12% of dataset to decrease debugging time # 10% was too small for a realistic calculation. choices = np.random.binomial(1, 0.12, len(training_data)) temp = [] predict_data = [] count = 0 for i in range(len(training_data)): if choices[i] == 1: temp.append(training_data[i]) count += 1 else: predict_data.append(training_data[i]) training_data = temp print('Training sample size: ', count, '/400\n', sep='') # Setting the default learner # Test simple learner svm learning_model = svm.SVC(probability=True, kernel='linear') learner = SimpleLearner(learning_model, training_data) learner.train() original_pred_labels = learner.predict(training_data) before_attack_label = original_pred_labels[0] orig_learner = deepcopy(learner) # Do the attack attacker = KInsertion(learner, training_data[0], number_to_add=number_to_add, num_iterations=num_iterations, verbose=True) attack_data = attacker.attack(training_data) # Retrain the model with poisoned data learning_model = svm.SVC(probability=True, kernel='linear') learner = SimpleLearner(learning_model, attack_data) learner.train() print('Number of added instances: ', len(attack_data) - len(training_data)) ############################################################################ # Calculate statistics with training data attack_pred_labels = learner.predict( training_data) # predict w/ orig label after_attack_label = attack_pred_labels[0] (orig_precent_correct, attack_precent_correct, difference) = calculate_correct_percentages(original_pred_labels, attack_pred_labels, training_data) print( '###################################################################') print('Predictions with training dataset:') print('Original correct percentage: ', orig_precent_correct, '%') print('Attack correct percentage: ', attack_precent_correct, '%') print('Difference: ', difference, '%') ############################################################################ # Calculate statistics with predict data (other half of dataset) original_pred_labels = orig_learner.predict(predict_data) attack_pred_labels = learner.predict(predict_data) (orig_precent_correct, attack_precent_correct, difference) = calculate_correct_percentages(original_pred_labels, attack_pred_labels, predict_data) print( '###################################################################') print('Predictions with other half of dataset:') print('Original correct percentage: ', orig_precent_correct, '%') print('Attack correct percentage: ', attack_precent_correct, '%') print('Difference: ', difference, '%') ############################################################################ # Calculate statistics with predict data (other half of dataset) print( '###################################################################') print('Selected instance true label: ', training_data[0].get_label()) print('Selected instance predicted label BEFORE attack: ', before_attack_label) print('Selected instance predicted label AFTER attack: ', after_attack_label) ############################################################################ # Output loss calculations print( '###################################################################') print('poison_instance loss before attack: ', round(attacker.poison_loss_before, 4), '%') print('poison_instance loss after attack: ', round(attacker.poison_loss_after, 4), '%') print('poison_instance loss difference: ', round(attacker.poison_loss_after - attacker.poison_loss_before, 4), '%') print('\nEND k-insertion attack.') print( '#################################################################\n')
def test_data_modification(): print() print( '###################################################################') print('START data modification attack.\n') begin = time.time() # Data processing unit # The path is an index of 400 testing samples(raw email data). dataset = EmailDataset(path='./data_reader/data/raw/trec05p-1/test-400', binary=False, raw=True) training_data, predict_data = dataset.split({'train': 50, 'test': 50}) training_data = load_dataset(training_data) predict_data = load_dataset(predict_data) print('Training sample size: ', len(training_data), '/400\n', sep='') # Setting the default learner # Test simple learner svm orig_learning_model = svm.SVC(probability=True, kernel='linear') orig_learner = SimpleLearner(orig_learning_model, training_data) orig_learner.train() ############################################################################ # Calculate target theta, 1 -> spam, -1 -> ham; For the target theta # calculation, I am assuming I know which spam I want to be classified # as ham and the features I want to have a disproportionate effect on the # decision function calculation. For example, if feature #32 is something # that all of my spam has in common, I want to make the entry corresponding # to #32 (index 32 - 1 = 31) in target_theta to be disproportionately # negative so that when my spam is being classified, the 1 indicating that # feature #32 is present will be multiplied by a large negative number so as # to decrease the value of the decision function and hopefully make it # negative so as to classify my spam as ham. lnr = orig_learner.model.learner eye = np.eye(training_data[0].get_feature_count(), dtype=int) orig_theta = lnr.decision_function(eye) - lnr.intercept_[0] target_theta = deepcopy(orig_theta) spam_instances = [] for inst in training_data + predict_data: if inst.get_label() == 1: spam_instances.append(inst) spam_features, ham_features = get_spam_features(spam_instances) # Set features to recognize spam as ham for index in spam_features: target_theta[index] = -10 for index in ham_features: target_theta[index] = 0.01 print('Features selected: ', np.array(spam_features)) print('Number of features: ', len(spam_features)) ############################################################################ # Get original predictions original_pred_labels = orig_learner.predict(training_data) # Do the attack attacker = DataModification(orig_learner, target_theta, verbose=True) attack_data = attacker.attack(training_data) # Retrain the model with poisoned data learning_model = svm.SVC(probability=True, kernel='linear') learner = SimpleLearner(learning_model, attack_data) learner.train() ############################################################################ # Calculate statistics with training data attack_pred_labels = learner.predict( training_data) # predict w/ orig label (orig_precent_correct, attack_precent_correct, difference) = calculate_correct_percentages(original_pred_labels, attack_pred_labels, training_data) print( '###################################################################') print('Predictions with training dataset:') print('Original correct percentage: ', orig_precent_correct, '%') print('Attack correct percentage: ', attack_precent_correct, '%') print('Difference: ', difference, '%') ############################################################################ # Calculate statistics with predict data (other half of dataset) original_pred_labels = orig_learner.predict(predict_data) attack_pred_labels = learner.predict(predict_data) (orig_precent_correct, attack_precent_correct, difference) = calculate_correct_percentages(original_pred_labels, attack_pred_labels, predict_data) print( '###################################################################') print('Predictions with other half of dataset:') print('Original correct percentage: ', orig_precent_correct, '%') print('Attack correct percentage: ', attack_precent_correct, '%') print('Difference: ', difference, '%') ############################################################################ # Calculate statistics with predict data (other half of dataset) spam_pred_labels = learner.predict(spam_instances) spam_ham_count = sum(map(lambda x: 1 if x == -1 else 0, spam_pred_labels)) print( '###################################################################') print('Number of spam instances in original training set that were \n', 'classified as ham after the attack: ', spam_ham_count, '/', len(spam_instances), sep='') end = time.time() print('\nTotal time: ', round(begin - end, 2), 's', '\n', sep='') print('\nEND data modification attack.') print( '###################################################################') print()
def empty_learner(): return SimpleLearner()
def test_k_insertion(): """ Use as follows: python3 adlib/tests/adversaries/k_insertion_test.py NUMBER-TO-ADD """ print() print('###################################################################') print('START k-insertion attack.\n') begin = time.time() # Data processing unit # The path is an index of 400 testing samples(raw email data). dataset = EmailDataset(path='./data_reader/data/raw/trec05p-1/test-400', binary=False, raw=True) training_data, predict_data = dataset.split({'train': 20, 'test': 80}) training_data = load_dataset(training_data) predict_data = load_dataset(predict_data) print('Training sample size: ', len(training_data), '/400\n', sep='') if len(sys.argv) > 2: number_to_add = int(sys.argv[1]) else: number_to_add = int(0.25 * len(training_data)) # Setting the default learner # Test simple learner svm learning_model = svm.SVC(probability=True, kernel='linear') learner = SimpleLearner(learning_model, training_data) learner.train() original_pred_labels = learner.predict(training_data) before_attack_label = original_pred_labels[0] orig_learner = deepcopy(learner) # Do the attack attacker = KInsertion(learner, training_data[0], number_to_add=number_to_add, verbose=True) attack_data = attacker.attack(training_data) # Retrain the model with poisoned data learning_model = svm.SVC(probability=True, kernel='linear') learner = SimpleLearner(learning_model, attack_data) learner.train() print('Number of added instances: ', len(attack_data) - len(training_data)) ############################################################################ # Calculate statistics with training data attack_pred_labels = learner.predict(training_data) # predict w/ orig label after_attack_label = attack_pred_labels[0] (orig_precent_correct, attack_precent_correct, difference) = calculate_correct_percentages(original_pred_labels, attack_pred_labels, training_data) print('###################################################################') print('Predictions with training dataset:') print('Original correct percentage: ', orig_precent_correct, '%') print('Attack correct percentage: ', attack_precent_correct, '%') print('Difference: ', difference, '%') ############################################################################ # Calculate statistics with predict data (other half of dataset) original_pred_labels = orig_learner.predict(predict_data) attack_pred_labels = learner.predict(predict_data) (orig_precent_correct, attack_precent_correct, difference) = calculate_correct_percentages(original_pred_labels, attack_pred_labels, predict_data) print('###################################################################') print('Predictions with other half of dataset:') print('Original correct percentage: ', orig_precent_correct, '%') print('Attack correct percentage: ', attack_precent_correct, '%') print('Difference: ', difference, '%') ############################################################################ # Calculate statistics with predict data (other half of dataset) print('###################################################################') print('Selected instance true label: ', training_data[0].get_label()) print('Selected instance predicted label BEFORE attack: ', before_attack_label) print('Selected instance predicted label AFTER attack: ', after_attack_label) ############################################################################ # Output loss calculations print('###################################################################') print('poison_instance loss before attack: ', round(attacker.poison_loss_before, 4)) print('poison_instance loss after attack: ', round(attacker.poison_loss_after, 4)) print('poison_instance loss difference: ', round(attacker.poison_loss_after - attacker.poison_loss_before, 4)) end = time.time() print('\nTotal time: ', round(end - begin, 2), 's', '\n', sep='') print('\nEND k-insertion attack.') print('###################################################################') print()