def setUpClass(cls):
master_seed(SEED)
Model = get_model('MnistCnnV2')
model = Model()
logger.info('Starting %s data container...', NAME)
dc = DataContainer(DATASET_LIST[NAME], get_data_path())
dc()
cls.mc = ModelContainerPT(model, dc)
cls.mc.load(MODEL_FILE)
accuracy = cls.mc.evaluate(dc.x_test, dc.y_test)
logger.info('Accuracy on test set: %f', accuracy)
cls.attack = BIMContainer(
cls.mc,
eps=0.3,
eps_step=0.1,
max_iter=100,
targeted=False)
def main():
Model = get_model('MnistCnnV2')
classifier = Model()
dc = DataContainer(DATASET_LIST['MNIST'], get_data_path())
dc()
classifier_mc = ModelContainerPT(classifier, dc)
classifier_mc.load(MODEL_FILE)
accuracy = classifier_mc.evaluate(dc.x_test, dc.y_test)
print(f'Accuracy on test set: {accuracy}')
attack = BIMContainer(classifier_mc,
eps=0.3,
eps_step=0.1,
max_iter=100,
targeted=False)
adv_trainer = AdversarialTraining(classifier_mc, [attack])
adv_trainer.fit(max_epochs=5, batch_size=128, ratio=0.1)
discriminator = adv_trainer.get_def_model_container()
print(discriminator.accuracy_test)
def main():
data_name = 'MNIST'
set_logging('advTraining', data_name, True, True)
model_file = os.path.join('save', 'MnistCnnV2_MNIST_e50.pt')
Model = get_model('MnistCnnV2')
classifier = Model()
dc = DataContainer(DATASET_LIST[data_name], get_data_path())
dc()
classifier_mc = ModelContainerPT(classifier, dc)
classifier_mc.load(model_file)
accuracy = classifier_mc.evaluate(dc.x_test, dc.y_test)
logger.info('Accuracy on test set: %f', accuracy)
attack = BIMContainer(classifier_mc,
eps=0.3,
eps_step=0.1,
max_iter=100,
targeted=False)
adv_trainer = AdversarialTraining(classifier_mc, [attack])
# adv_trainer.fit(max_epochs=30, batch_size=128, ratio=0.1)
# adv_trainer.save('AdvTrain_MnistCnnV2_MNIST', overwrite=True)
file_name = os.path.join('save', 'AdvTrain_MnistCnnV2_MNIST.pt')
adv_trainer.load(file_name)
x = np.load(os.path.join('save', 'MnistCnnV2_MNIST_BIM_x.npy'),
allow_pickle=False)
y = np.load(os.path.join('save', 'MnistCnnV2_MNIST_BIM_y.npy'),
allow_pickle=False)
blocked_indices = adv_trainer.detect(x, return_passed_x=False)
logger.info('Blocked %d/%d samples on %s', len(blocked_indices), len(x),
'clean')
adv = np.load(os.path.join('save', 'MnistCnnV2_MNIST_BIM_adv.npy'),
allow_pickle=False)
accuracy = classifier_mc.evaluate(adv, y)
logger.info('Accuracy on %s set: %f', 'BIM', accuracy)
blocked_indices = adv_trainer.detect(adv, return_passed_x=False)
logger.info('Blocked %d/%d samples on %s', len(blocked_indices), len(adv),
'BIM')
adv = np.load(os.path.join('save', 'MnistCnnV2_MNIST_Carlini_adv.npy'),
allow_pickle=False)
accuracy = classifier_mc.evaluate(adv, y)
logger.info('Accuracy on %s set: %f', 'Carlini', accuracy)
blocked_indices = adv_trainer.detect(adv, return_passed_x=False)
logger.info('Blocked %d/%d samples on %s', len(blocked_indices), len(adv),
'Carlini')
adv = np.load(os.path.join('save', 'MnistCnnV2_MNIST_DeepFool_adv.npy'),
allow_pickle=False)
accuracy = classifier_mc.evaluate(adv, y)
logger.info('Accuracy on %s set: %f', 'DeepFool', accuracy)
blocked_indices = adv_trainer.detect(adv, return_passed_x=False)
logger.info('Blocked %d/%d samples on %s', len(blocked_indices), len(adv),
'DeepFool')
adv = np.load(os.path.join('save', 'MnistCnnV2_MNIST_FGSM_adv.npy'),
allow_pickle=False)
accuracy = classifier_mc.evaluate(adv, y)
logger.info('Accuracy on %s set: %f', 'FGSM', accuracy)
blocked_indices = adv_trainer.detect(adv, return_passed_x=False)
logger.info('Blocked %d/%d samples on %s', len(blocked_indices), len(adv),
'FGSM')
adv = np.load(os.path.join('save', 'MnistCnnV2_MNIST_Saliency_adv.npy'),
allow_pickle=False)
accuracy = classifier_mc.evaluate(adv, y)
logger.info('Accuracy on %s set: %f', 'Saliency', accuracy)
blocked_indices = adv_trainer.detect(adv, return_passed_x=False)
logger.info('Blocked %d/%d samples on %s', len(blocked_indices), len(adv),
'Saliency')
def experiment(index, data_container, max_epochs, adv_file, res_file):
# STEP 1: select data and model
dname = data_container.name
num_classes = data_container.num_classes
num_features = data_container.dim_data[0]
model = IrisNN(num_features=num_features,
hidden_nodes=num_features * 4,
num_classes=num_classes)
distill_model = IrisNN(num_features=num_features,
hidden_nodes=num_features * 4,
num_classes=num_classes)
# STEP 2: train models
mc = ModelContainerPT(model, data_container)
mc.fit(max_epochs=max_epochs, batch_size=BATCH_SIZE)
accuracy = mc.evaluate(data_container.x_test, data_container.y_test)
logger.info('Accuracy on test set: %f', accuracy)
adv_res = [accuracy]
# STEP 3: generate adversarial examples
# no more than 1000 samples are required
x = data_container.x_test
y = data_container.y_test
# The test set has fixed size, 1000.
if len(x) > 1000:
x = x[:1000]
y = y[:1000]
accuracy = mc.evaluate(x, y)
adv_res.append(accuracy)
advs = np.zeros((len(ATTACK_LIST), x.shape[0], x.shape[1]),
dtype=np.float32)
pred_advs = -np.ones((len(ATTACK_LIST), len(y)),
dtype=np.int32) # assign -1 as initial value
pred_clean = mc.predict(x)
advs[0] = x
pred_advs[0] = pred_clean
att_param_json = open(os.path.join(DIR_PATH, 'AttackParams.json'))
att_params = json.load(att_param_json)
for i, att_name in enumerate(ATTACK_LIST):
# Clean set is only used in evaluation phase.
if att_name == 'Clean':
continue
logger.debug('[%d]Running %s attack...', i, att_name)
kwargs = att_params[att_name]
logger.debug('%s params: %s', att_name, str(kwargs))
Attack = get_attack(att_name)
attack = Attack(mc, **kwargs)
adv, pred_adv, x_clean, pred_clean_ = attack.generate(
use_testset=False, x=x)
assert np.all(pred_clean == pred_clean_)
assert np.all(x == x_clean)
logger.info('created %d adv examples using %s from %s', len(advs[i]),
att_name, dname)
not_match = pred_adv != pred_clean
success_rate = len(not_match[not_match == True]) / len(pred_clean)
accuracy = mc.evaluate(adv, y)
advs[i] = adv
pred_advs[i] = pred_adv
logger.info('Success rate of %s: %f', att_name, success_rate)
logger.info('Accuracy on %s: %f', att_name, accuracy)
adv_res.append(accuracy)
adv_file.write(','.join([str(r) for r in adv_res]) + '\n')
# STEP 4: train defences
blocked_res = np.zeros(len(TITLE_RESULTS), dtype=np.int32)
blocked_res[0] = index
for def_name in DEFENCE_LIST:
logger.debug('Running %s...', def_name)
if def_name == 'AdvTraining':
attack = BIMContainer(mc,
eps=0.3,
eps_step=0.1,
max_iter=100,
targeted=False)
defence = AdversarialTraining(mc, [attack])
defence.fit(max_epochs=max_epochs,
batch_size=BATCH_SIZE,
ratio=ADV_TRAIN_RATIO)
block_attack(0, advs, defence, def_name, blocked_res)
elif def_name == 'Destillation':
# A single temperature is used for all sets
temp = 20
defence = DistillationContainer(mc,
distill_model,
temperature=temp,
pretrained=False)
defence.fit(max_epochs=max_epochs, batch_size=BATCH_SIZE)
block_attack(1, advs, defence, def_name, blocked_res)
elif def_name == 'Squeezing':
defence = FeatureSqueezing(
mc,
SQUEEZER_FILTER_LIST,
bit_depth=SQUEEZER_DEPTH,
sigma=SQUEEZER_SIGMA,
pretrained=True,
)
defence.fit(max_epochs=max_epochs, batch_size=BATCH_SIZE)
block_attack(2, advs, defence, def_name, blocked_res)
elif def_name == 'AD':
ad_param_file = open(AD_PARAM_FILE)
# BreastCancer uses a different set of parameters
if dname == 'BreastCancerWisconsin':
param_file = os.path.join(DIR_PATH, 'AdParamsBC.json')
ad_param_file = open(param_file)
ad_params = json.load(ad_param_file)
logger.debug('AD params: %s', str(ad_params))
defence = ApplicabilityDomainContainer(
mc, hidden_model=model.hidden_model, **ad_params)
defence.fit()
block_attack(3, advs, defence, def_name, blocked_res)
res_file.write(','.join([str(r) for r in blocked_res]) + '\n')
def main():
parser = ap.ArgumentParser()
parser.add_argument(
'-m', '--model', type=str, required=True,
help='a file which contains a pretrained model. The filename should in "<model>_<dataset>_e<max epochs>[_<date>].pt" format')
parser.add_argument(
'-p', '--param', type=str, required=True,
help='a JSON config file which contains the parameters for the cross validation')
parser.add_argument(
'-a', '--adv', type=str,
help='file name of adv. examples for testing. If it\'s none, the program will ignore testing. The name should in "<model>_<dataset>_<attack>_adv.npy" format')
parser.add_argument(
'-s', '--seed', type=int, default=4096,
help='the seed for random number generator')
parser.add_argument(
'-v', '--verbose', action='store_true', default=False,
help='set logger level to debug')
parser.add_argument(
'-l', '--savelog', action='store_true', default=False,
help='save logging file')
parser.add_argument(
'-i', '--ignore', action='store_true', default=False,
help='Ignore saving the results. Only returns the results from terminal.')
parser.add_argument(
'-w', '--overwrite', action='store_true', default=False,
help='overwrite the existing file')
args = parser.parse_args()
model_file = args.model
param_file = args.param
adv_file = args.adv
seed = args.seed
verbose = args.verbose
save_log = args.savelog
does_ignore = args.ignore
overwrite = args.overwrite
model_name, data_name = parse_model_filename(model_file)
# set logging config. Run this before logging anything!
set_logging('cross_validation', data_name, verbose, save_log)
# check files
for file_path in [model_file, param_file]:
if not os.path.exists(file_path):
logger.warning('%s does not exist. Exit.', file_path)
sys.exit(0)
if adv_file is not None and not os.path.exists(adv_file):
logger.warning('%s does not exist. Exit.', adv_file)
sys.exit(0)
# read parameters
with open(param_file) as param_json:
params = json.load(param_json)
# show parameters
print('[cv] Running cross validation on {} with {}...'.format(
model_file, data_name))
logger.info('Start at : %s', get_time_str())
logger.info('RECEIVED PARAMETERS:')
logger.info('model file :%s', model_file)
logger.info('adv file :%s', adv_file)
logger.info('model :%s', model_name)
logger.info('dataset :%s', data_name)
logger.info('param file :%s', param_file)
logger.info('seed :%d', seed)
logger.info('verbose :%r', verbose)
logger.info('save_log :%r', save_log)
logger.info('Ignore saving :%r', does_ignore)
logger.info('overwrite :%r', overwrite)
logger.debug('params :%s', str(params))
# load parameters
k_range = params['k_range']
z_range = params['z_range']
kappa_range = params['kappa_range']
gamma_range = params['gamma_range']
epsilon = params['epsilon']
num_folds = params['num_folds']
batch_size = params['batch_size']
sample_ratio = params['sample_ratio']
logger.info('k_range :%s', str(k_range))
logger.info('z_range :%s', str(z_range))
logger.info('kappa_range :%s', str(kappa_range))
logger.info('gamma_range :%s', str(gamma_range))
logger.info('epsilon :%.1f', epsilon)
logger.info('num_folds :%d', num_folds)
logger.info('batch_size :%d', batch_size)
logger.info('sample_ratio :%.1f', sample_ratio)
# reset seed
master_seed(seed)
dc = DataContainer(DATASET_LIST[data_name], get_data_path())
dc(shuffle=True, normalize=True, size_train=0.8)
logger.info('Sample size: %d', len(dc))
Model = get_model(model_name)
# there models require extra keyword arguments
if data_name in ('BankNote', 'HTRU2', 'Iris', 'WheatSeed'):
num_classes = dc.num_classes
num_features = dc.dim_data[0]
kwargs = {
'num_features': num_features,
'hidden_nodes': num_features*4,
'num_classes': num_classes,
}
model = Model(**kwargs)
else:
model = Model()
logger.info('Use %s model', model.__class__.__name__)
mc = ModelContainerPT(model, dc)
mc.load(model_file)
accuracy = mc.evaluate(dc.x_test, dc.y_test)
logger.info('Accuracy on test set: %f', accuracy)
ad = ApplicabilityDomainContainer(
mc, hidden_model=model.hidden_model, sample_ratio=sample_ratio)
cross_validation = CrossValidation(
ad,
num_folds=num_folds,
k_range=k_range,
z_range=z_range,
kappa_range=kappa_range,
gamma_range=gamma_range,
epsilon=epsilon,
)
bim_attack = BIMContainer(
mc,
eps=0.3,
eps_step=0.1,
max_iter=100,
targeted=False,
)
cross_validation.fit(bim_attack)
# test optimal parameters
if adv_file is not None:
postfix = ['adv', 'pred', 'x', 'y']
data_files = [adv_file.replace('_adv', '_' + s) for s in postfix]
adv = np.load(data_files[0], allow_pickle=False)
pred_adv = np.load(data_files[1], allow_pickle=False)
x = np.load(data_files[2], allow_pickle=False)
pred = np.load(data_files[3], allow_pickle=False)
# fetch optimal parameters
ad = ApplicabilityDomainContainer(
mc,
hidden_model=model.hidden_model,
k2=cross_validation.k2,
reliability=cross_validation.reliability,
sample_ratio=sample_ratio,
kappa=cross_validation.kappa,
confidence=cross_validation.confidence,
)
logger.info('Params: %s', str(ad.params))
ad.fit()
blocked_indices = ad.detect(x, pred, return_passed_x=False)
logger.info('Blocked %d/%d on clean data',
len(blocked_indices), len(x))
blocked_indices = ad.detect(adv, pred_adv, return_passed_x=False)
logger.info('Blocked %d/%d on adv. examples.',
len(blocked_indices), len(adv))
# save results
if not does_ignore:
file_name = name_handler(
model_name + '_' + data_name, 'csv', overwrite=overwrite)
cross_validation.save(file_name)
def experiment(index, dname, mname, max_epochs, adv_file, res_file):
# STEP 1: select data
dc = get_data_container(dname, use_shuffle=True, use_normalize=True)
Model = get_model(mname)
model = Model()
distill_model = Model()
logger.info('Selected %s model', model.__class__.__name__)
# STEP 2: train models
mc = ModelContainerPT(model, dc)
mc.fit(max_epochs=max_epochs, batch_size=BATCH_SIZE)
accuracy = mc.evaluate(dc.x_test, dc.y_test)
logger.info('Accuracy on test set: %f', accuracy)
adv_res = [accuracy]
# STEP 3: generate adversarial examples
# no more than 1000 samples are required
n = 1000 if len(dc.x_test) >= 1000 else len(dc.x_test)
# idx = np.random.choice(len(dc.x_test), n, replace=False)
# x = dc.x_test[idx]
# y = dc.y_test[idx]
x = dc.x_test[:n]
y = dc.y_test[:n]
accuracy = mc.evaluate(x, y)
adv_res.append(accuracy)
advs = np.zeros(
tuple([len(ATTACK_LIST)] + list(x.shape)),
dtype=np.float32)
pred_advs = -np.ones(
(len(ATTACK_LIST), n),
dtype=np.int32) # assign -1 as initial value
pred_clean = mc.predict(x)
advs[0] = x
pred_advs[0] = pred_clean
att_param_json = open(os.path.join(DIR_PATH, 'AttackParams.json'))
att_params = json.load(att_param_json)
for i, att_name in enumerate(ATTACK_LIST):
# Clean set is only used in evaluation phase.
if att_name == 'Clean':
continue
logger.debug('[%d]Running %s attack...', i, att_name)
kwargs = att_params[att_name]
logger.debug('%s params: %s', att_name, str(kwargs))
Attack = get_attack(att_name)
attack = Attack(mc, **kwargs)
adv, pred_adv, x_clean, pred_clean_ = attack.generate(
use_testset=False,
x=x)
assert np.all(pred_clean == pred_clean_)
assert np.all(x == x_clean)
logger.info('created %d adv examples using %s from %s',
len(advs[i]),
att_name,
dname)
not_match = pred_adv != pred_clean
success_rate = len(not_match[not_match == True]) / len(pred_clean)
accuracy = mc.evaluate(adv, y)
advs[i] = adv
pred_advs[i] = pred_adv
logger.info('Success rate of %s: %f', att_name, success_rate)
logger.info('Accuracy on %s: %f', att_name, accuracy)
adv_res.append(accuracy)
adv_file.write(','.join([str(r) for r in adv_res]) + '\n')
# STEP 4: train defences
blocked_res = np.zeros(len(TITLE_RESULTS), dtype=np.int32)
blocked_res[0] = index
for def_name in DEFENCE_LIST:
logger.debug('Running %s...', def_name)
if def_name == 'AdvTraining':
attack = BIMContainer(
mc,
eps=0.3,
eps_step=0.1,
max_iter=100,
targeted=False)
defence = AdversarialTraining(mc, [attack])
defence.fit(max_epochs=max_epochs,
batch_size=BATCH_SIZE,
ratio=ADV_TRAIN_RATIO)
block_attack(0, advs, defence, def_name, blocked_res)
elif def_name == 'Destillation':
defence = DistillationContainer(
mc, distill_model, temperature=DISTILL_TEMP, pretrained=False)
defence.fit(max_epochs=max_epochs, batch_size=BATCH_SIZE)
block_attack(1, advs, defence, def_name, blocked_res)
elif def_name == 'Squeezing':
defence = FeatureSqueezing(
mc,
SQUEEZER_FILTER_LIST,
bit_depth=SQUEEZER_DEPTH,
sigma=SQUEEZER_SIGMA,
kernel_size=SQUEEZER_KERNEL,
pretrained=True,
)
defence.fit(max_epochs=max_epochs, batch_size=BATCH_SIZE)
block_attack(2, advs, defence, def_name, blocked_res)
elif def_name == 'AD':
ad_param_file = open(AD_PARAM_FILE)
ad_params = json.load(ad_param_file)
logger.debug('AD params: %s', str(ad_params))
defence = ApplicabilityDomainContainer(
mc,
hidden_model=model.hidden_model,
**ad_params)
defence.fit()
block_attack(3, advs, defence, def_name, blocked_res)
res_file.write(','.join([str(r) for r in blocked_res]) + '\n')
def main():
parser = ap.ArgumentParser()
parser.add_argument(
'-m', '--model', type=str, required=True,
help='a file which contains a pretrained model. The filename should in "<model>_<dataset>_e<max epochs>[_<date>].pt" format')
parser.add_argument(
'-e', '--epoch', type=int, required=True,
help='the number of max epochs for training')
parser.add_argument(
'-r', '--ratio', type=float, required=True,
help='the percentage of adversarial examples mix to the training set.')
parser.add_argument(
'-b', '--batchsize', type=int, default=128, help='batch size')
parser.add_argument(
'-t', '--train', action='store_true', default=False,
help='Force the model to retrain without searching existing pretrained file')
parser.add_argument(
'-s', '--seed', type=int, default=4096,
help='the seed for random number generator')
parser.add_argument(
'-v', '--verbose', action='store_true', default=False,
help='set logger level to debug')
parser.add_argument(
'-l', '--savelog', action='store_true', default=False,
help='save logging file')
parser.add_argument(
'-B', '--bim', action='store_true', default=False,
help='Apply BIM attack')
parser.add_argument(
'-C', '--carlini', action='store_true', default=False,
help='Apply Carlini L2 attack')
parser.add_argument(
'-D', '--deepfool', action='store_true', default=False,
help='Apply DeepFool attack')
parser.add_argument(
'-F', '--fgsm', action='store_true', default=False,
help='Apply FGSM attack')
parser.add_argument(
'-S', '--saliency', action='store_true', default=False,
help='Apply Saliency Map attack')
args = parser.parse_args()
model_file = args.model
max_epochs = args.epoch
ratio = args.ratio
batch_size = args.batchsize
seed = args.seed
verbose = args.verbose
save_log = args.savelog
need_train = args.train
model_name, data_name = parse_model_filename(model_file)
# Which attack should apply?
attack_list = []
if args.bim:
attack_list.append('BIM')
if args.carlini:
attack_list.append('Carlini')
if args.deepfool:
attack_list.append('DeepFool')
if args.fgsm:
attack_list.append('FGSM')
if args.saliency:
attack_list.append('Saliency')
# Quit, if there is nothing to do.
if len(attack_list) == 0:
logger.warning('Neither received any filter nor any attack. Exit')
sys.exit(0)
y_file = os.path.join(
'save', f'{model_name}_{data_name}_{attack_list[0]}_y.npy')
attack_files = [
os.path.join(
'save', f'{model_name}_{data_name}_{attack_list[0]}_x.npy')
]
for attack_name in attack_list:
attack_files.append(os.path.join(
'save', f'{model_name}_{data_name}_{attack_name}_adv.npy'))
# the 1st file this the clean inputs
attack_list = ['clean'] + attack_list
# Do I need train the discriminator?
pretrain_file = f'AdvTrain_{model_name}_{data_name}.pt'
if not os.path.exists(os.path.join('save', pretrain_file)):
need_train = True
# set logging config. Run this before logging anything!
set_logging(LOG_NAME, data_name, verbose, save_log)
# show parameters
print(f'[{LOG_NAME}] Running adversarial training on {model_name}...')
logger.info('Start at : %s', get_time_str())
logger.info('RECEIVED PARAMETERS:')
logger.info('model file :%s', model_file)
logger.info('model :%s', model_name)
logger.info('dataset :%s', data_name)
logger.info('max_epochs :%d', max_epochs)
logger.info('ratio :%d', ratio)
logger.info('batch_size :%d', batch_size)
logger.info('seed :%d', seed)
logger.info('verbose :%r', verbose)
logger.info('save_log :%r', save_log)
logger.info('need train :%r', need_train)
logger.info('attacks :%s', ', '.join(attack_list))
# check files
for file_name in [model_file, y_file] + attack_files:
if not os.path.exists(file_name):
logger.error('%s does not exist!', file_name)
raise FileNotFoundError('{} does not exist!'.format(file_name))
# reset seed
master_seed(seed)
# select data
dc = get_data_container(
data_name,
use_shuffle=True,
use_normalize=True,
)
# select a model
Model = get_model(model_name)
model = Model()
if data_name in ('BankNote', 'HTRU2', 'Iris', 'WheatSeed'):
num_classes = dc.num_classes
num_features = dc.dim_data[0]
model = IrisNN(
num_features=num_features,
hidden_nodes=num_features*4,
num_classes=num_classes)
classifier_mc = ModelContainerPT(model, dc)
classifier_mc.load(model_file)
accuracy = classifier_mc.evaluate(dc.x_test, dc.y_test)
logger.info('Accuracy on test set: %f', accuracy)
attack = BIMContainer(
classifier_mc,
eps=0.3,
eps_step=0.1,
max_iter=100,
targeted=False)
adv_trainer = AdversarialTraining(classifier_mc, [attack])
if need_train:
adv_trainer.fit(max_epochs=max_epochs,
batch_size=batch_size, ratio=ratio)
adv_trainer.save(pretrain_file, overwrite=True)
else:
adv_trainer.load(os.path.join('save', pretrain_file))
y = np.load(y_file, allow_pickle=False)
for i in range(len(attack_list)):
adv_file = attack_files[i]
adv_name = attack_list[i]
logger.debug('Load %s...', adv_file)
adv = np.load(adv_file, allow_pickle=False)
accuracy = classifier_mc.evaluate(adv, y)
logger.info('Accuracy on %s set: %f', adv_name, accuracy)
blocked_indices = adv_trainer.detect(adv, return_passed_x=False)
logger.info('Blocked %d/%d samples on %s',
len(blocked_indices), len(adv), adv_name)
def main():
data_name = 'Iris'
set_logging('advTraining', data_name, True, True)
dc = DataContainer(DATASET_LIST[data_name], get_data_path())
dc()
model_file = os.path.join('save', 'IrisNN_Iris_e200.pt')
num_features = dc.dim_data[0]
num_classes = dc.num_classes
classifier = IrisNN(
num_features=num_features,
hidden_nodes=num_features*4,
num_classes=num_classes,
)
classifier_mc = ModelContainerPT(classifier, dc)
classifier_mc.load(model_file)
accuracy = classifier_mc.evaluate(dc.x_test, dc.y_test)
logger.info('Accuracy on test set: %f', accuracy)
attack = BIMContainer(
classifier_mc,
eps=0.3,
eps_step=0.1,
max_iter=100,
targeted=False)
adv_trainer = AdversarialTraining(classifier_mc, [attack])
# adv_trainer.fit(max_epochs=100, batch_size=64, ratio=1)
# adv_trainer.save('AdvTrain_IrisNN_Iris', overwrite=True)
file_name = os.path.join('save', 'AdvTrain_IrisNN_Iris.pt')
adv_trainer.load(file_name)
x = np.load(os.path.join('save', 'IrisNN_Iris_BIM_x.npy'),
allow_pickle=False)
y = np.load(os.path.join('save', 'IrisNN_Iris_BIM_y.npy'),
allow_pickle=False)
blocked_indices = adv_trainer.detect(x, return_passed_x=False)
logger.info('Blocked %d/%d samples on %s',
len(blocked_indices), len(x), 'clean')
adv = np.load(os.path.join(
'save', 'IrisNN_Iris_BIM_adv.npy'), allow_pickle=False)
accuracy = classifier_mc.evaluate(adv, y)
logger.info('Accuracy on %s set: %f', 'BIM', accuracy)
blocked_indices = adv_trainer.detect(adv, return_passed_x=False)
logger.info('Blocked %d/%d samples on %s',
len(blocked_indices), len(adv), 'BIM')
adv = np.load(os.path.join(
'save', 'IrisNN_Iris_Carlini_adv.npy'), allow_pickle=False)
accuracy = classifier_mc.evaluate(adv, y)
logger.info('Accuracy on %s set: %f', 'Carlini', accuracy)
blocked_indices = adv_trainer.detect(adv, return_passed_x=False)
logger.info('Blocked %d/%d samples on %s',
len(blocked_indices), len(adv), 'Carlini')
adv = np.load(os.path.join(
'save', 'IrisNN_Iris_DeepFool_adv.npy'), allow_pickle=False)
accuracy = classifier_mc.evaluate(adv, y)
logger.info('Accuracy on %s set: %f', 'DeepFool', accuracy)
blocked_indices = adv_trainer.detect(adv, return_passed_x=False)
logger.info('Blocked %d/%d samples on %s',
len(blocked_indices), len(adv), 'DeepFool')
adv = np.load(os.path.join(
'save', 'IrisNN_Iris_FGSM_adv.npy'), allow_pickle=False)
accuracy = classifier_mc.evaluate(adv, y)
logger.info('Accuracy on %s set: %f', 'FGSM', accuracy)
blocked_indices = adv_trainer.detect(adv, return_passed_x=False)
logger.info('Blocked %d/%d samples on %s',
len(blocked_indices), len(adv), 'FGSM')