def setUpClass(cls):
master_seed(SEED)
Model = get_model('MnistCnnV2')
model = Model()
logger.info('Starting %s data container...', NAME)
dc = DataContainer(DATASET_LIST[NAME], get_data_path())
dc()
mc = ModelContainerPT(model, dc)
mc.load(MODEL_FILE)
accuracy = mc.evaluate(dc.x_test, dc.y_test)
logger.info('Accuracy on test set: %f', accuracy)
cls.distillation = DistillationContainer(mc,
Model(),
temperature=TEMPERATURE,
pretrained=False)
filename = get_pt_model_filename(
model.__class__.__name__, NAME,
str(MAX_EPOCHS) + 't' + str(int(TEMPERATURE * 10)))
filename = os.path.join('test', 'distill_' + filename)
file_path = os.path.join('save', filename)
if not os.path.exists(file_path):
# Expected initial loss = -log(1/num_classes) = 2.3025850929940455'
cls.distillation.fit(max_epochs=MAX_EPOCHS, batch_size=BATCH_SIZE)
cls.distillation.save(filename, overwrite=True)
else:
cls.distillation.load(file_path)
smooth_mc = cls.distillation.get_def_model_container()
accuracy = smooth_mc.evaluate(dc.x_test, dc.y_test)
logger.info('Accuracy on test set: %f', accuracy)
def main():
master_seed(SEED)
logger.info('Starting %s data container...', NAME)
dc = DataContainer(DATASET_LIST[NAME], get_data_path())
dc(shuffle=True, normalize=True)
num_features = dc.dim_data[0]
num_classes = dc.num_classes
print('Features:', num_features)
print('Classes:', num_classes)
model = BCNN(num_features, num_classes)
filename = get_pt_model_filename(BCNN.__name__, NAME, MAX_EPOCHS)
logger.debug('File name: %s', filename)
mc = ModelContainerPT(model, dc)
file_path = os.path.join('save', filename)
if not os.path.exists(file_path):
logger.debug('Expected initial loss: %f', np.log(dc.num_classes))
mc.fit(max_epochs=MAX_EPOCHS, batch_size=BATCH_SIZE)
mc.save(filename, overwrite=True)
else:
logger.info('Use saved parameters from %s', filename)
mc.load(file_path)
accuracy = mc.evaluate(dc.x_test, dc.y_test)
logger.info('Accuracy on test set: %f', accuracy)
def main():
master_seed(SEED)
dataset = DATASET_LIST[DATA_NAME]
dc = DataContainer(dataset, get_data_path())
dc()
model = MnistCnnCW()
mc = ModelContainerPT(model, dc)
mc.load(MODEL_FILE)
accuracy = mc.evaluate(dc.x_test, dc.y_test)
print('Accuracy on test set: {}'.format(accuracy))
attack = CarliniL2V2Container(mc,
targeted=False,
learning_rate=0.01,
binary_search_steps=9,
max_iter=1000,
confidence=0.0,
initial_const=0.01,
c_range=(0, 1e10),
batch_size=16,
clip_values=(0.0, 1.0))
adv, y_adv, x_clean, y_clean = attack.generate(count=100)
l2 = np.mean(get_l2_norm(adv, x_clean))
print('L2 norm: {}'.format(l2))
not_match = y_adv != y_clean
success_rate = len(not_match[not_match == True]) / len(adv)
print('Success rate: {}'.format(success_rate))
accuracy = mc.evaluate(adv, y_clean)
print('Accuracy on adv. examples: {}'.format(accuracy))
def test_master_seed_np(self):
master_seed(SEED)
a = np.random.rand(10)
b = np.random.rand(10)
master_seed(SEED)
z = np.random.rand(10)
self.assertTrue((a != b).any())
self.assertTrue((z == a).all())
def test_get_range(self):
x = np.array([[2, 2], [0, 0], [1, 1]], dtype=np.float32)
out = get_range(x)
e = ([0., 0.], [2., 2.])
r = [all(xx == ee) for xx, ee in zip(out, e)]
self.assertTrue(all(r))
master_seed(SEED)
x = np.random.rand(10, 1, 28, 28)
out = np.array(list(get_range(x, is_image=True)))
np.testing.assert_almost_equal(out, [0., 0.9998], decimal=4)
def test_master_seed_torch(self):
import torch
master_seed(SEED)
a = torch.randn(10)
b = torch.randn(10)
master_seed(SEED)
z = torch.randn((10))
self.assertTrue((a != b).any())
self.assertTrue((z == a).all())
def setUpClass(cls):
master_seed(SEED)
Model = get_model('MnistCnnV2')
model = Model()
logger.info('Starting %s data container...', NAME)
dc = DataContainer(DATASET_LIST[NAME], get_data_path())
dc()
cls.mc = ModelContainerPT(model, dc)
cls.mc.load(MODEL_FILE)
accuracy = cls.mc.evaluate(dc.x_test, dc.y_test)
logger.info('Accuracy on test set: %f', accuracy)
def setUpClass(cls):
master_seed(SEED)
logger.info('Starting %s data container...', NAME)
cls.dc = DataContainer(DATASET_LIST[NAME], get_data_path())
cls.dc(shuffle=True)
model = MnistCnnV2()
logger.info('Using model: %s', model.__class__.__name__)
cls.mc = ModelContainerPT(model, cls.dc)
filename = get_pt_model_filename(MnistCnnV2.__name__, NAME, MAX_EPOCHS)
file_path = os.path.join('save', filename)
if not os.path.exists(file_path):
cls.mc.fit(max_epochs=MAX_EPOCHS, batch_size=BATCH_SIZE)
cls.mc.save(filename, overwrite=True)
else:
logger.info('Use saved parameters from %s', filename)
cls.mc.load(file_path)
accuracy = cls.mc.evaluate(cls.dc.x_test, cls.dc.y_test)
logger.info('Accuracy on test set: %f', accuracy)
hidden_model = model.hidden_model
logger.info('sample_ratio: %f', SAMPLE_RATIO)
cls.ad = ApplicabilityDomainContainer(
cls.mc,
hidden_model=hidden_model,
k2=9,
reliability=1.6,
sample_ratio=SAMPLE_RATIO,
kappa=10,
confidence=0.9,
)
cls.ad.fit()
# shuffle the test set
x_test = cls.dc.x_test
y_test = cls.dc.y_test
shuffled_indices = np.random.permutation(len(x_test))[:NUM_ADV]
cls.x = x_test[shuffled_indices]
cls.y = y_test[shuffled_indices]
logger.info('# of test set: %d', len(cls.x))
def setUpClass(cls):
master_seed(SEED)
logger.info('Starting %s data container...', NAME)
cls.dc = DataContainer(DATASET_LIST[NAME], get_data_path())
cls.dc(shuffle=True)
model = CifarCnn()
logger.info('Using model: %s', model.__class__.__name__)
cls.mc = ModelContainerPT(model, cls.dc)
file_path = os.path.join('save', FILE_NAME)
if not os.path.exists(file_path):
raise Exception('Where is the pretrained file?!')
else:
logger.info('Use saved parameters from %s', FILE_NAME)
cls.mc.load(file_path)
accuracy = cls.mc.evaluate(cls.dc.x_test, cls.dc.y_test)
logger.info('Accuracy on test set: %f', accuracy)
hidden_model = model.hidden_model
logger.info('sample_ratio: %f', SAMPLE_RATIO)
cls.ad = ApplicabilityDomainContainer(
cls.mc,
hidden_model=hidden_model,
k2=9,
reliability=1.6,
sample_ratio=SAMPLE_RATIO,
kappa=10,
confidence=0.7,
)
cls.ad.fit()
# shuffle the test set
x_test = cls.dc.x_test
y_test = cls.dc.y_test
shuffled_indices = np.random.permutation(len(x_test))[:NUM_ADV]
cls.x = x_test[shuffled_indices]
cls.y = y_test[shuffled_indices]
logger.info('# of test set: %d', len(cls.x))
def main():
master_seed(SEED)
dataset = DATASET_LIST[DATA_NAME]
dc = DataContainer(dataset, get_data_path())
dc(shuffle=True, normalize=True)
print('# of trainset: {}, # of testset: {}'.format(len(dc.x_train),
len(dc.x_test)))
num_classes = dc.num_classes
num_features = dc.dim_data[0]
model = IrisNN(num_features=num_features,
hidden_nodes=num_features * 4,
num_classes=num_classes)
mc = ModelContainerPT(model, dc)
mc.load(MODEL_FILE)
accuracy = mc.evaluate(dc.x_test, dc.y_test)
print('Accuracy on test set: {}'.format(accuracy))
clip_values = get_range(dc.x_train, is_image=False)
print('clip_values', clip_values)
attack = CarliniL2V2Container(mc,
targeted=False,
learning_rate=0.01,
binary_search_steps=9,
max_iter=1000,
confidence=0.0,
initial_const=0.01,
c_range=(0, 1e4),
batch_size=16,
clip_values=clip_values)
adv, y_adv, x_clean, y_clean = attack.generate(count=100)
l2 = np.mean(get_l2_norm(adv, x_clean))
print('L2 norm: {}'.format(l2))
not_match = y_adv != y_clean
success_rate = len(not_match[not_match == True]) / len(adv)
print('Success rate: {}'.format(success_rate))
accuracy = mc.evaluate(adv, y_clean)
print('Accuracy on adv. examples: {}'.format(accuracy))
def setUpClass(cls):
master_seed(SEED)
logger.info('Starting %s data container...', NAME)
cls.dc = DataContainer(DATASET_LIST[NAME], get_data_path())
cls.dc(shuffle=False)
model = MnistCnnCW()
logger.info('Using model: %s', model.__class__.__name__)
cls.mc = ModelContainerPT(model, cls.dc)
file_path = os.path.join('save', FILE_NAME)
if not os.path.exists(file_path):
cls.mc.fit(max_epochs=MAX_EPOCHS, batch_size=BATCH_SIZE)
cls.mc.save(FILE_NAME, overwrite=True)
else:
logger.info('Use saved parameters from %s', FILE_NAME)
cls.mc.load(file_path)
accuracy = cls.mc.evaluate(cls.dc.x_test, cls.dc.y_test)
logger.info('Accuracy on test set: %f', accuracy)
def setUpClass(cls):
master_seed(SEED)
logger.info('Starting %s data container...', NAME)
cls.dc = DataContainer(DATASET_LIST[NAME], get_data_path())
# ordered by labels, it requires shuffle!
cls.dc(shuffle=True, normalize=True)
num_features = cls.dc.dim_data[0]
num_classes = cls.dc.num_classes
model = BCNN(num_features, num_classes)
logger.info('Using model: %s', model.__class__.__name__)
cls.mc = ModelContainerPT(model, cls.dc)
filename = get_pt_model_filename(BCNN.__name__, NAME, MAX_EPOCHS)
file_path = os.path.join('save', filename)
if not os.path.exists(file_path):
cls.mc.fit(max_epochs=MAX_EPOCHS, batch_size=BATCH_SIZE)
cls.mc.save(filename, overwrite=True)
else:
logger.info('Use saved parameters from %s', filename)
cls.mc.load(file_path)
accuracy = cls.mc.evaluate(cls.dc.x_test, cls.dc.y_test)
logger.info('Accuracy on test set: %f', accuracy)
hidden_model = model.hidden_model
cls.ad = ApplicabilityDomainContainer(
cls.mc,
hidden_model=hidden_model,
k2=6,
reliability=1.6,
sample_ratio=SAMPLE_RATIO,
kappa=10,
confidence=0.9,
)
cls.ad.fit()
def setUpClass(cls):
master_seed(SEED)
# generating synthetic data
x, y = make_classification(
n_samples=SAMPLE_SIZE,
n_features=NUM_FEATURES,
n_informative=NUM_CLASSES,
n_redundant=0,
n_classes=NUM_CLASSES,
n_clusters_per_class=1,
)
x_max = np.max(x, axis=0)
x_min = np.min(x, axis=0)
x = scale_normalize(x, x_min, x_max)
n_train = int(np.floor(SAMPLE_SIZE * 0.8))
x_train = np.array(x[:n_train], dtype=np.float32)
y_train = np.array(y[:n_train], dtype=np.long)
x_test = np.array(x[n_train:], dtype=np.float32)
y_test = np.array(y[n_train:], dtype=np.long)
data_dict = get_synthetic_dataset_dict(SAMPLE_SIZE, NUM_CLASSES,
NUM_FEATURES)
dc = DataContainer(data_dict, get_data_path())
dc.x_train = x_train
dc.y_train = y_train
dc.x_test = x_test
dc.y_test = y_test
# training Extra Tree classifier
classifier = ExtraTreeClassifier(
criterion='gini',
splitter='random',
)
cls.mc = ModelContainerTree(classifier, dc)
cls.mc.fit()
accuracy = cls.mc.evaluate(dc.x_test, dc.y_test)
logger.info('Accuracy on test set: %f', accuracy)
def setUpClass(cls):
master_seed(SEED)
if not os.path.exists(os.path.join('save', 'test')):
os.makedirs(os.path.join('save', 'test'))
NAME = 'Iris'
logger.info('Starting %s data container...', NAME)
cls.dc = DataContainer(DATASET_LIST[NAME], get_data_path())
cls.dc(shuffle=True)
model = IrisNN()
model_name = model.__class__.__name__
logger.info('Using model: %s', model_name)
cls.mc = ModelContainerPT(model, cls.dc)
cls.mc.fit(max_epochs=100, batch_size=BATCH_SIZE)
# for comparison
model2 = IrisNN()
cls.mc2 = ModelContainerPT(model2, cls.dc)
# inputs for testing
cls.x = np.copy(cls.dc.x_test[:5])
cls.y = np.copy(cls.dc.y_test[:5])
def main():
parser = ap.ArgumentParser()
parser.add_argument(
'-m', '--model', type=str, required=True,
help='a file which contains a pretrained model. The filename should in "<model>_<dataset>_e<max epochs>[_<date>].pt" format')
parser.add_argument(
'-p', '--param', type=str, required=True,
help='a JSON config file which contains the parameters for the cross validation')
parser.add_argument(
'-a', '--adv', type=str,
help='file name of adv. examples for testing. If it\'s none, the program will ignore testing. The name should in "<model>_<dataset>_<attack>_adv.npy" format')
parser.add_argument(
'-s', '--seed', type=int, default=4096,
help='the seed for random number generator')
parser.add_argument(
'-v', '--verbose', action='store_true', default=False,
help='set logger level to debug')
parser.add_argument(
'-l', '--savelog', action='store_true', default=False,
help='save logging file')
parser.add_argument(
'-i', '--ignore', action='store_true', default=False,
help='Ignore saving the results. Only returns the results from terminal.')
parser.add_argument(
'-w', '--overwrite', action='store_true', default=False,
help='overwrite the existing file')
args = parser.parse_args()
model_file = args.model
param_file = args.param
adv_file = args.adv
seed = args.seed
verbose = args.verbose
save_log = args.savelog
does_ignore = args.ignore
overwrite = args.overwrite
model_name, data_name = parse_model_filename(model_file)
# set logging config. Run this before logging anything!
set_logging('cross_validation', data_name, verbose, save_log)
# check files
for file_path in [model_file, param_file]:
if not os.path.exists(file_path):
logger.warning('%s does not exist. Exit.', file_path)
sys.exit(0)
if adv_file is not None and not os.path.exists(adv_file):
logger.warning('%s does not exist. Exit.', adv_file)
sys.exit(0)
# read parameters
with open(param_file) as param_json:
params = json.load(param_json)
# show parameters
print('[cv] Running cross validation on {} with {}...'.format(
model_file, data_name))
logger.info('Start at : %s', get_time_str())
logger.info('RECEIVED PARAMETERS:')
logger.info('model file :%s', model_file)
logger.info('adv file :%s', adv_file)
logger.info('model :%s', model_name)
logger.info('dataset :%s', data_name)
logger.info('param file :%s', param_file)
logger.info('seed :%d', seed)
logger.info('verbose :%r', verbose)
logger.info('save_log :%r', save_log)
logger.info('Ignore saving :%r', does_ignore)
logger.info('overwrite :%r', overwrite)
logger.debug('params :%s', str(params))
# load parameters
k_range = params['k_range']
z_range = params['z_range']
kappa_range = params['kappa_range']
gamma_range = params['gamma_range']
epsilon = params['epsilon']
num_folds = params['num_folds']
batch_size = params['batch_size']
sample_ratio = params['sample_ratio']
logger.info('k_range :%s', str(k_range))
logger.info('z_range :%s', str(z_range))
logger.info('kappa_range :%s', str(kappa_range))
logger.info('gamma_range :%s', str(gamma_range))
logger.info('epsilon :%.1f', epsilon)
logger.info('num_folds :%d', num_folds)
logger.info('batch_size :%d', batch_size)
logger.info('sample_ratio :%.1f', sample_ratio)
# reset seed
master_seed(seed)
dc = DataContainer(DATASET_LIST[data_name], get_data_path())
dc(shuffle=True, normalize=True, size_train=0.8)
logger.info('Sample size: %d', len(dc))
Model = get_model(model_name)
# there models require extra keyword arguments
if data_name in ('BankNote', 'HTRU2', 'Iris', 'WheatSeed'):
num_classes = dc.num_classes
num_features = dc.dim_data[0]
kwargs = {
'num_features': num_features,
'hidden_nodes': num_features*4,
'num_classes': num_classes,
}
model = Model(**kwargs)
else:
model = Model()
logger.info('Use %s model', model.__class__.__name__)
mc = ModelContainerPT(model, dc)
mc.load(model_file)
accuracy = mc.evaluate(dc.x_test, dc.y_test)
logger.info('Accuracy on test set: %f', accuracy)
ad = ApplicabilityDomainContainer(
mc, hidden_model=model.hidden_model, sample_ratio=sample_ratio)
cross_validation = CrossValidation(
ad,
num_folds=num_folds,
k_range=k_range,
z_range=z_range,
kappa_range=kappa_range,
gamma_range=gamma_range,
epsilon=epsilon,
)
bim_attack = BIMContainer(
mc,
eps=0.3,
eps_step=0.1,
max_iter=100,
targeted=False,
)
cross_validation.fit(bim_attack)
# test optimal parameters
if adv_file is not None:
postfix = ['adv', 'pred', 'x', 'y']
data_files = [adv_file.replace('_adv', '_' + s) for s in postfix]
adv = np.load(data_files[0], allow_pickle=False)
pred_adv = np.load(data_files[1], allow_pickle=False)
x = np.load(data_files[2], allow_pickle=False)
pred = np.load(data_files[3], allow_pickle=False)
# fetch optimal parameters
ad = ApplicabilityDomainContainer(
mc,
hidden_model=model.hidden_model,
k2=cross_validation.k2,
reliability=cross_validation.reliability,
sample_ratio=sample_ratio,
kappa=cross_validation.kappa,
confidence=cross_validation.confidence,
)
logger.info('Params: %s', str(ad.params))
ad.fit()
blocked_indices = ad.detect(x, pred, return_passed_x=False)
logger.info('Blocked %d/%d on clean data',
len(blocked_indices), len(x))
blocked_indices = ad.detect(adv, pred_adv, return_passed_x=False)
logger.info('Blocked %d/%d on adv. examples.',
len(blocked_indices), len(adv))
# save results
if not does_ignore:
file_name = name_handler(
model_name + '_' + data_name, 'csv', overwrite=overwrite)
cross_validation.save(file_name)
def setUp(self):
master_seed(SEED)
def main():
parser = ap.ArgumentParser()
parser.add_argument(
'-m',
'--model',
type=str,
required=True,
help=
'a file which contains a pretrained model. The filename should in "<model>_<dataset>_e<max epochs>[_<date>].pt" format'
)
parser.add_argument('-e',
'--epoch',
type=int,
required=True,
help='the number of max epochs for training')
parser.add_argument(
'-d',
'--depth',
type=int,
default=0,
help=
'The image color depth for input images. Apply Binary-Depth filter when receives a parameter'
)
parser.add_argument(
'--sigma',
type=float,
default=0,
help=
'The Standard Deviation of Normal distribution. Apply Gaussian Noise filter when receives a parameter'
)
parser.add_argument(
'-k',
'--kernelsize',
type=int,
default=0,
help=
'The kernel size for Median filter. Apply median filter when receives a parameter'
)
parser.add_argument('-b',
'--batchsize',
type=int,
default=128,
help='batch size')
parser.add_argument(
'-T',
'--train',
action='store_true',
default=False,
help=
'Force the model to retrain without searching existing pretrained file'
)
parser.add_argument('-s',
'--seed',
type=int,
default=4096,
help='the seed for random number generator')
parser.add_argument('-v',
'--verbose',
action='store_true',
default=False,
help='set logger level to debug')
parser.add_argument('-l',
'--savelog',
action='store_true',
default=False,
help='save logging file')
parser.add_argument('-B',
'--bim',
action='store_true',
default=False,
help='Apply BIM attack')
parser.add_argument('-C',
'--carlini',
action='store_true',
default=False,
help='Apply Carlini L2 attack')
parser.add_argument('-D',
'--deepfool',
action='store_true',
default=False,
help='Apply DeepFool attack')
parser.add_argument('-F',
'--fgsm',
action='store_true',
default=False,
help='Apply FGSM attack')
parser.add_argument('-S',
'--saliency',
action='store_true',
default=False,
help='Apply Saliency Map attack')
args = parser.parse_args()
model_file = args.model
max_epochs = args.epoch
bit_depth = args.depth
sigma = args.sigma
kernel_size = args.kernelsize
batch_size = args.batchsize
seed = args.seed
verbose = args.verbose
save_log = args.savelog
need_train = args.train
model_name, data_name = parse_model_filename(model_file)
# Which filter should apply?
filter_list = []
if bit_depth > 0:
filter_list.append('binary')
if sigma > 0:
filter_list.append('normal')
if kernel_size > 0:
filter_list.append('median')
# Which attack should apply?
attack_list = []
if args.fgsm:
attack_list.append('FGSM')
if args.bim:
attack_list.append('BIM')
if args.deepfool:
attack_list.append('DeepFool')
if args.carlini:
attack_list.append('Carlini')
if args.saliency:
attack_list.append('Saliency')
# Quit, if there is nothing to do.
if len(filter_list) == 0 or len(attack_list) == 0:
logger.warning('Neither received any filter nor any attack. Exit')
sys.exit(0)
y_file = os.path.join('save',
f'{model_name}_{data_name}_{attack_list[0]}_y.npy')
attack_files = [
os.path.join('save',
f'{model_name}_{data_name}_{attack_list[0]}_x.npy')
]
for attack_name in attack_list:
attack_files.append(
os.path.join('save',
f'{model_name}_{data_name}_{attack_name}_adv.npy'))
# the 1st file this the clean inputs
attack_list = ['clean'] + attack_list
# Do I need train the distillation network?
pretrain_files = []
for fname in filter_list:
pretrain_file = build_squeezer_filename(model_name, data_name,
max_epochs, fname)
pretrain_files.append(pretrain_file)
if not os.path.exists(os.path.join('save', pretrain_file)):
need_train = True
# set logging config. Run this before logging anything!
set_logging(LOG_NAME, data_name, verbose, save_log)
# show parameters
print(f'[{LOG_NAME}] Running feature squeezing on {model_name}...')
logger.info('Start at : %s', get_time_str())
logger.info('RECEIVED PARAMETERS:')
logger.info('model file :%s', model_file)
logger.info('model :%s', model_name)
logger.info('dataset :%s', data_name)
logger.info('max_epochs :%d', max_epochs)
logger.info('bit_depth :%d', bit_depth)
logger.info('sigma :%f', sigma)
logger.info('kernel_size :%d', kernel_size)
logger.info('batch_size :%d', batch_size)
logger.info('seed :%d', seed)
logger.info('verbose :%r', verbose)
logger.info('save_log :%r', save_log)
logger.info('need train :%r', need_train)
logger.info('filters :%s', ', '.join(filter_list))
logger.info('attacks :%s', ', '.join(attack_list))
logger.info('pretrained :%s', ', '.join(pretrain_files))
# check files
for file_name in [model_file, y_file] + attack_files:
if not os.path.exists(file_name):
logger.error('%s does not exist!', file_name)
raise FileNotFoundError('{} does not exist!'.format(file_name))
# reset seed
master_seed(seed)
# select data
dc = get_data_container(
data_name,
use_shuffle=True,
use_normalize=True,
)
# select a model
Model = get_model(model_name)
model = Model()
if data_name in ('BankNote', 'HTRU2', 'Iris', 'WheatSeed'):
num_classes = dc.num_classes
num_features = dc.dim_data[0]
model = IrisNN(num_features=num_features,
hidden_nodes=num_features * 4,
num_classes=num_classes)
classifier_mc = ModelContainerPT(model, dc)
classifier_mc.load(model_file)
accuracy = classifier_mc.evaluate(dc.x_test, dc.y_test)
logger.info('Accuracy on test set: %f', accuracy)
# initialize Squeezer
squeezer = FeatureSqueezing(
classifier_mc,
filter_list,
bit_depth=bit_depth,
sigma=sigma,
kernel_size=kernel_size,
pretrained=True,
)
# train or load parameters for Squeezer
if need_train:
squeezer.fit(max_epochs=max_epochs, batch_size=batch_size)
squeezer.save(model_file, True)
else:
squeezer.load(model_file)
# traverse all attacks
y = np.load(y_file, allow_pickle=False)
for i in range(len(attack_list)):
adv_file = attack_files[i]
adv_name = attack_list[i]
logger.debug('Load %s...', adv_file)
adv = np.load(adv_file, allow_pickle=False)
acc_og = classifier_mc.evaluate(adv, y)
acc_squeezer = squeezer.evaluate(adv, y)
logger.info('Accuracy on %s set - OG: %f, Squeezer: %f', adv_name,
acc_og, acc_squeezer)
blocked_indices = squeezer.detect(adv, return_passed_x=False)
logger.info('Blocked %d/%d samples on %s', len(blocked_indices),
len(adv), adv_name)
eps=0.3,
eps_step=0.1,
max_iter=100,
targeted=False,
)
k2, zeta, kappa, gamma = cross_validation(dc, model, bim_attack)
# use pre-defined parameters
# k2, zeta, kappa, gamma, score = K2, RELIABILITY, KAPPA, CONFIDENCE, 0
ad = ApplicabilityDomainContainer(
mc,
hidden_model=model.hidden_model,
k2=k2,
reliability=zeta,
sample_ratio=SAMPLE_RATIO,
kappa=kappa,
confidence=gamma,
)
print(ad.params)
ad.fit()
blocked_indices, x_passed = ad.detect(adv, y_adv, return_passed_x=True)
print('After update parameters, blocked {}/{} samples from adv. examples'.
format(len(blocked_indices), len(adv)))
if __name__ == '__main__':
master_seed(SEED)
main()
def main():
parser = ap.ArgumentParser()
parser.add_argument(
'-m', '--model', type=str, required=True,
help='a file which contains a pretrained model. The filename should in "<model>_<dataset>_e<max epochs>[_<date>].pt" format')
parser.add_argument(
'-p', '--param', type=str, required=True,
help='a JSON config file which contains the parameters for the attacks')
parser.add_argument(
'-n', '--number', type=int, default=1000,
help='the number of adv. examples want to generate. (if more than test set, it uses all test examples.)')
parser.add_argument(
'-s', '--seed', type=int, default=4096,
help='the seed for random number generator')
parser.add_argument(
'-v', '--verbose', action='store_true', default=False,
help='set logger level to debug')
parser.add_argument(
'-l', '--savelog', action='store_true', default=False,
help='save logging file')
parser.add_argument(
'-w', '--overwrite', action='store_true', default=False,
help='overwrite the existing file')
parser.add_argument(
'-B', '--bim', action='store_true', default=False,
help='Apply BIM attack')
parser.add_argument(
'-C', '--carlini', action='store_true', default=False,
help='Apply Carlini L2 attack')
parser.add_argument(
'-D', '--deepfool', action='store_true', default=False,
help='Apply DeepFool attack')
parser.add_argument(
'-F', '--fgsm', action='store_true', default=False,
help='Apply FGSM attack')
parser.add_argument(
'-S', '--saliency', action='store_true', default=False,
help='Apply Saliency Map attack')
args = parser.parse_args()
model_file = args.model
attack_param_file = args.param
num_adv = args.number
seed = args.seed
verbose = args.verbose
save_log = args.savelog
overwrite = args.overwrite
# Which attack should apply?
# use binary encoding for attacks
my_attacks = np.zeros(5, dtype=np.int8)
attack_list = np.array(
['FGSM', 'BIM', 'Carlini', 'DeepFool', 'Saliency'])
my_attacks[0] = 1 if args.fgsm else 0
my_attacks[1] = 1 if args.bim else 0
my_attacks[2] = 1 if args.carlini else 0
my_attacks[3] = 1 if args.deepfool else 0
my_attacks[4] = 1 if args.saliency else 0
selected_attacks = attack_list[np.where(my_attacks == 1)[0]]
# check file
for f in [model_file, attack_param_file]:
if not os.path.exists(f):
raise FileNotFoundError('{} does not exist!'.format(f))
dirname = os.path.dirname(model_file)
model_name, dname = parse_model_filename(model_file)
with open(attack_param_file) as param_json:
att_params = json.load(param_json)
# set logging config. Run this before logging anything!
set_logging('attack', dname, verbose, save_log)
# show parameters
print('[attack] Start generating {} adv. samples from {} model...'.format(
num_adv, model_name))
logger.info('Start at : %s', get_time_str())
logger.info('RECEIVED PARAMETERS:')
logger.info('model file :%s', model_file)
logger.info('model :%s', model_name)
logger.info('dataset :%s', dname)
logger.info('params :%s', attack_param_file)
logger.info('num_adv :%r', num_adv)
logger.info('seed :%d', seed)
logger.info('verbose :%r', verbose)
logger.info('save_log :%r', save_log)
logger.info('overwrite :%r', overwrite)
logger.info('dirname :%r', dirname)
logger.info('attacks :%s', ', '.join(selected_attacks))
if len(selected_attacks) == 0:
logger.warning('No attack is selected. Exit.')
sys.exit(0)
# reset seed
master_seed(seed)
# set DataContainer and ModelContainer
dc = get_data_container(dname)
Model = get_model(model_name)
# there models require extra keyword arguments
if dname in ('BankNote', 'HTRU2', 'Iris', 'WheatSeed'):
num_classes = dc.num_classes
num_features = dc.dim_data[0]
kwargs = {
'num_features': num_features,
'hidden_nodes': num_features*4,
'num_classes': num_classes,
}
model = Model(**kwargs)
else:
model = Model()
logger.info('Use %s model', model.__class__.__name__)
mc = ModelContainerPT(model, dc)
mc.load(model_file)
accuracy = mc.evaluate(dc.x_test, dc.y_test)
logger.info('Accuracy on test set: %f', accuracy)
run_attacks(mc,
selected_attacks,
att_params,
num_adv,
model_name + '_' + dname,
overwrite)
def main():
parser = ap.ArgumentParser()
parser.add_argument(
'-m', '--model', type=str, required=True,
help='a file which contains a pretrained model. The filename should in "<model>_<dataset>_e<max epochs>[_<date>].pt" format')
parser.add_argument(
'-e', '--epoch', type=int, required=True,
help='the number of max epochs for training')
parser.add_argument(
'-r', '--ratio', type=float, required=True,
help='the percentage of adversarial examples mix to the training set.')
parser.add_argument(
'-b', '--batchsize', type=int, default=128, help='batch size')
parser.add_argument(
'-t', '--train', action='store_true', default=False,
help='Force the model to retrain without searching existing pretrained file')
parser.add_argument(
'-s', '--seed', type=int, default=4096,
help='the seed for random number generator')
parser.add_argument(
'-v', '--verbose', action='store_true', default=False,
help='set logger level to debug')
parser.add_argument(
'-l', '--savelog', action='store_true', default=False,
help='save logging file')
parser.add_argument(
'-B', '--bim', action='store_true', default=False,
help='Apply BIM attack')
parser.add_argument(
'-C', '--carlini', action='store_true', default=False,
help='Apply Carlini L2 attack')
parser.add_argument(
'-D', '--deepfool', action='store_true', default=False,
help='Apply DeepFool attack')
parser.add_argument(
'-F', '--fgsm', action='store_true', default=False,
help='Apply FGSM attack')
parser.add_argument(
'-S', '--saliency', action='store_true', default=False,
help='Apply Saliency Map attack')
args = parser.parse_args()
model_file = args.model
max_epochs = args.epoch
ratio = args.ratio
batch_size = args.batchsize
seed = args.seed
verbose = args.verbose
save_log = args.savelog
need_train = args.train
model_name, data_name = parse_model_filename(model_file)
# Which attack should apply?
attack_list = []
if args.bim:
attack_list.append('BIM')
if args.carlini:
attack_list.append('Carlini')
if args.deepfool:
attack_list.append('DeepFool')
if args.fgsm:
attack_list.append('FGSM')
if args.saliency:
attack_list.append('Saliency')
# Quit, if there is nothing to do.
if len(attack_list) == 0:
logger.warning('Neither received any filter nor any attack. Exit')
sys.exit(0)
y_file = os.path.join(
'save', f'{model_name}_{data_name}_{attack_list[0]}_y.npy')
attack_files = [
os.path.join(
'save', f'{model_name}_{data_name}_{attack_list[0]}_x.npy')
]
for attack_name in attack_list:
attack_files.append(os.path.join(
'save', f'{model_name}_{data_name}_{attack_name}_adv.npy'))
# the 1st file this the clean inputs
attack_list = ['clean'] + attack_list
# Do I need train the discriminator?
pretrain_file = f'AdvTrain_{model_name}_{data_name}.pt'
if not os.path.exists(os.path.join('save', pretrain_file)):
need_train = True
# set logging config. Run this before logging anything!
set_logging(LOG_NAME, data_name, verbose, save_log)
# show parameters
print(f'[{LOG_NAME}] Running adversarial training on {model_name}...')
logger.info('Start at : %s', get_time_str())
logger.info('RECEIVED PARAMETERS:')
logger.info('model file :%s', model_file)
logger.info('model :%s', model_name)
logger.info('dataset :%s', data_name)
logger.info('max_epochs :%d', max_epochs)
logger.info('ratio :%d', ratio)
logger.info('batch_size :%d', batch_size)
logger.info('seed :%d', seed)
logger.info('verbose :%r', verbose)
logger.info('save_log :%r', save_log)
logger.info('need train :%r', need_train)
logger.info('attacks :%s', ', '.join(attack_list))
# check files
for file_name in [model_file, y_file] + attack_files:
if not os.path.exists(file_name):
logger.error('%s does not exist!', file_name)
raise FileNotFoundError('{} does not exist!'.format(file_name))
# reset seed
master_seed(seed)
# select data
dc = get_data_container(
data_name,
use_shuffle=True,
use_normalize=True,
)
# select a model
Model = get_model(model_name)
model = Model()
if data_name in ('BankNote', 'HTRU2', 'Iris', 'WheatSeed'):
num_classes = dc.num_classes
num_features = dc.dim_data[0]
model = IrisNN(
num_features=num_features,
hidden_nodes=num_features*4,
num_classes=num_classes)
classifier_mc = ModelContainerPT(model, dc)
classifier_mc.load(model_file)
accuracy = classifier_mc.evaluate(dc.x_test, dc.y_test)
logger.info('Accuracy on test set: %f', accuracy)
attack = BIMContainer(
classifier_mc,
eps=0.3,
eps_step=0.1,
max_iter=100,
targeted=False)
adv_trainer = AdversarialTraining(classifier_mc, [attack])
if need_train:
adv_trainer.fit(max_epochs=max_epochs,
batch_size=batch_size, ratio=ratio)
adv_trainer.save(pretrain_file, overwrite=True)
else:
adv_trainer.load(os.path.join('save', pretrain_file))
y = np.load(y_file, allow_pickle=False)
for i in range(len(attack_list)):
adv_file = attack_files[i]
adv_name = attack_list[i]
logger.debug('Load %s...', adv_file)
adv = np.load(adv_file, allow_pickle=False)
accuracy = classifier_mc.evaluate(adv, y)
logger.info('Accuracy on %s set: %f', adv_name, accuracy)
blocked_indices = adv_trainer.detect(adv, return_passed_x=False)
logger.info('Blocked %d/%d samples on %s',
len(blocked_indices), len(adv), adv_name)
def main():
parser = ap.ArgumentParser()
parser.add_argument('-d',
'--dataset',
type=str,
required=True,
help='Name of the dataset')
parser.add_argument(
'-p',
'--param',
type=str,
required=True,
help='a JSON config file which contains the parameters for the attacks'
)
parser.add_argument('-s',
'--seed',
type=int,
default=4096,
help='the seed for random number generator')
parser.add_argument('-v',
'--verbose',
action='store_true',
default=False,
help='set logger level to debug')
parser.add_argument('-l',
'--savelog',
action='store_true',
default=False,
help='save logging file')
parser.add_argument('-F',
'--fgsm',
action='store_true',
default=False,
help='Apply FGSM attack')
parser.add_argument('-B',
'--bim',
action='store_true',
default=False,
help='Apply BIM attack')
parser.add_argument('-D',
'--deepfool',
action='store_true',
default=False,
help='Apply DeepFool attack')
parser.add_argument('-C',
'--carlini',
action='store_true',
default=False,
help='Apply Carlini L2 attack')
args = parser.parse_args()
data_name = args.dataset
param_file = args.param
seed = args.seed
verbose = args.verbose
save_log = args.savelog
# set logging config. Run this before logging anything!
set_logging(LOG_NAME, data_name, verbose, save_log)
# Which attack should apply?
attack_list = []
if args.fgsm:
attack_list.append('FGSM')
if args.bim:
attack_list.append('BIM')
if args.deepfool:
attack_list.append('DeepFool')
if args.carlini:
attack_list.append('Carlini')
# Quit, if there is nothing to do.
if len(attack_list) == 0:
logger.warning('Neither received any filter nor any attack. Exit')
sys.exit(0)
if data_name in ('BankNote', 'HTRU2', 'Iris', 'WheatSeed'):
model_name = 'IrisNN'
if data_name == 'BreastCancerWisconsin':
model_name = 'BCNN'
y_file = os.path.join('save',
f'{model_name}_{data_name}_{attack_list[0]}_y.npy')
attack_files = [
os.path.join('save',
f'{model_name}_{data_name}_{attack_list[0]}_x.npy')
]
for attack_name in attack_list:
attack_files.append(
os.path.join('save',
f'{model_name}_{data_name}_{attack_name}_adv.npy'))
# the 1st file this the clean inputs
attack_list = ['clean'] + attack_list
# load parameters for Applicability Domain
with open(param_file) as param_json:
params = json.load(param_json)
# show parameters
print(f'[{LOG_NAME}] Running tree model...')
logger.info('Start at :%s', get_time_str())
logger.info('RECEIVED PARAMETERS:')
logger.info('model :%s', model_name)
logger.info('dataset :%s', data_name)
logger.info('param file :%s', param_file)
logger.info('seed :%d', seed)
logger.info('verbose :%r', verbose)
logger.info('save_log :%r', save_log)
logger.info('attacks :%s', ', '.join(attack_list))
logger.debug('params :%s', str(params))
# check files
for file_name in [y_file] + attack_files:
if not os.path.exists(file_name):
logger.error('%s does not exist!', file_name)
raise FileNotFoundError('{} does not exist!'.format(file_name))
# reset seed
master_seed(seed)
# select data
dc = get_data_container(
data_name,
use_shuffle=True,
use_normalize=True,
)
# train the model
classifier = ExtraTreeClassifier(
criterion='gini',
splitter='random',
)
mc = ModelContainerTree(classifier, dc)
mc.fit()
x = np.load(attack_files[0], allow_pickle=False)
art_classifier = SklearnClassifier(classifier)
attack = DecisionTreeAttack(art_classifier)
adv = attack.generate(x)
ad = ApplicabilityDomainContainer(mc, mc.hidden_model, **params)
ad.fit()
# generate adversarial examples
y = np.load(y_file, allow_pickle=False)
accuracy = mc.evaluate(adv, y)
logger.info('Accuracy on DecisionTreeAttack set: %f', accuracy)
blocked_indices = ad.detect(adv)
logger.info('Blocked %d/%d samples on DecisionTreeAttack',
len(blocked_indices), len(adv))
# traverse other attacks
for i in range(len(attack_list)):
adv_file = attack_files[i]
adv_name = attack_list[i]
logger.debug('Load %s...', adv_file)
adv = np.load(adv_file, allow_pickle=False)
accuracy = mc.evaluate(adv, y)
logger.info('Accuracy on %s set: %f', adv_name, accuracy)
blocked_indices = ad.detect(adv, return_passed_x=False)
logger.info('Blocked %d/%d samples on %s', len(blocked_indices),
len(adv), adv_name)
def main():
parser = ap.ArgumentParser()
parser.add_argument(
'-a',
'--adv',
type=str,
required=True,
help=
'file name for adv. examples. The name should in "<model>_<dataset>_<attack>_adv.npy" format'
)
parser.add_argument(
'-p',
'--param',
type=str,
required=True,
help='a JSON config file which contains the parameters for the attacks'
)
parser.add_argument(
'-m',
'--model',
type=str,
required=True,
help=
'a file which contains a pretrained model. The filename should in "<model>_<dataset>_e<max epochs>[_<date>].pt" format'
)
parser.add_argument('-s',
'--seed',
type=int,
default=4096,
help='the seed for random number generator')
parser.add_argument('-v',
'--verbose',
action='store_true',
default=False,
help='set logger level to debug')
parser.add_argument('-l',
'--savelog',
action='store_true',
default=False,
help='save logging file')
args = parser.parse_args()
adv_file = args.adv
param_file = args.param
model_file = args.model
seed = args.seed
verbose = args.verbose
save_log = args.savelog
check_clean = True
# build filenames from the root file
postfix = ['adv', 'pred', 'x', 'y']
data_files = [adv_file.replace('_adv', '_' + s) for s in postfix]
model_name, dname = parse_model_filename(adv_file)
# set logging config. Run this before logging anything!
set_logging('defence_ad', dname, verbose, save_log)
# check adv. examples and parameter config files
for f in data_files[:2] + [param_file]:
if not os.path.exists(f):
logger.warning('%s does not exist. Exit.', f)
sys.exit(0)
# check clean samples
for f in data_files[-2:]:
if not os.path.exists(f):
logger.warning(
'Cannot load files for clean samples. Skip checking clean set.'
)
check_clean = False
with open(param_file) as param_json:
params = json.load(param_json)
# show parameters
print(
'[defend_ad] Running applicability domain on {}...'.format(model_name))
logger.info('Start at : %s', get_time_str())
logger.info('RECEIVED PARAMETERS:')
logger.info('model file :%s', model_file)
logger.info('adv file :%s', adv_file)
logger.info('model :%s', model_name)
logger.info('dataset :%s', dname)
logger.info('param file :%s', param_file)
logger.info('seed :%d', seed)
logger.info('verbose :%r', verbose)
logger.info('save_log :%r', save_log)
logger.info('check_clean :%r', check_clean)
logger.debug('params : %s', str(params))
# reset seed
master_seed(seed)
# set DataContainer and ModelContainer
dc = get_data_container(dname)
Model = get_model(model_name)
# there models require extra keyword arguments
if dname in ('BankNote', 'HTRU2', 'Iris', 'WheatSeed'):
num_classes = dc.num_classes
num_features = dc.dim_data[0]
kwargs = {
'num_features': num_features,
'hidden_nodes': num_features * 4,
'num_classes': num_classes,
}
model = Model(**kwargs)
else:
model = Model()
logger.info('Use %s model', model.__class__.__name__)
mc = ModelContainerPT(model, dc)
mc.load(model_file)
accuracy = mc.evaluate(dc.x_test, dc.y_test)
logger.info('Accuracy on test set: %f', accuracy)
# preform defence
ad = ApplicabilityDomainContainer(mc,
hidden_model=model.hidden_model,
**params)
ad.fit()
result_prefix = [model_file] \
+ [adv_file] \
+ [params['k2']] \
+ [params['reliability']] \
+ [params['sample_ratio']] \
+ [params['confidence']] \
+ [params['kappa']] \
+ [params['disable_s2']]
# check clean
if check_clean:
x = np.load(data_files[2], allow_pickle=False)
y = np.load(data_files[3], allow_pickle=False)
x_passed, blk_idx, blocked_counts = detect(ad, 'clean samples', x, y)
result = result_prefix + ['clean'] + blocked_counts
result_clean = '[result]' + ','.join([str(r) for r in result])
# check adversarial examples
adv = np.load(data_files[0], allow_pickle=False)
pred = np.load(data_files[1], allow_pickle=False)
adv_passed, adv_blk_idx, blocked_counts = detect(ad, 'adv. examples', adv,
pred)
result = result_prefix + ['adv'] + blocked_counts
result = '[result]' + ','.join([str(r) for r in result])
if check_clean:
logger.info(result_clean)
logger.info(result)
def main():
parser = ap.ArgumentParser()
parser.add_argument('-d',
'--dataset',
type=str,
required=True,
choices=get_dataset_list(),
help='the dataset you want to train')
parser.add_argument(
'-o',
'--ofile',
type=str,
help='the filename will be used to store model parameters')
parser.add_argument('-e',
'--epoch',
type=int,
default=5,
help='the number of max epochs for training')
parser.add_argument('-b',
'--batchsize',
type=int,
default=128,
help='batch size')
parser.add_argument('-s',
'--seed',
type=int,
default=4096,
help='the seed for random number generator')
parser.add_argument('-H',
'--shuffle',
type=bool,
default=True,
help='shuffle the dataset')
parser.add_argument(
'-n',
'--normalize',
type=bool,
default=True,
help=
'apply zero mean and scaling to the dataset (for numeral dataset only)'
)
parser.add_argument('-m',
'--model',
type=str,
choices=AVALIABLE_MODELS,
help='select a model to train the data')
parser.add_argument('-v',
'--verbose',
action='store_true',
default=False,
help='set logger level to debug')
parser.add_argument('-l',
'--savelog',
action='store_true',
default=False,
help='save logging file')
parser.add_argument('-w',
'--overwrite',
action='store_true',
default=False,
help='overwrite the existing file')
args = parser.parse_args()
dname = args.dataset
filename = args.ofile
max_epochs = args.epoch
batch_size = args.batchsize
seed = args.seed
use_shuffle = args.shuffle
use_normalize = args.normalize
model_name = args.model
verbose = args.verbose
save_log = args.savelog
overwrite = args.overwrite
# set logging config. Run this before logging anything!
set_logging('train', dname, verbose, save_log)
# show parameters
print('[train] Start training {} model...'.format(model_name))
logger.info('Start at : %s', get_time_str())
logger.info('RECEIVED PARAMETERS:')
logger.info('dataset :%s', dname)
logger.info('filename :%s', filename)
logger.info('max_epochs :%d', max_epochs)
logger.info('batch_size :%d', batch_size)
logger.info('seed :%d', seed)
logger.info('use_shuffle :%r', use_shuffle)
logger.info('use_normalize :%r', use_normalize)
logger.info('model_name :%s', model_name)
logger.info('verbose :%r', verbose)
logger.info('save_log :%r', save_log)
logger.info('overwrite :%r', overwrite)
master_seed(seed)
# set DataContainer
dc = get_data_container(
dname,
use_shuffle=use_shuffle,
use_normalize=use_normalize,
)
# select a model
model = None
if model_name is not None:
Model = models.get_model(model_name)
model = Model()
else:
if dname == 'MNIST':
model = models.MnistCnnV2()
elif dname == 'CIFAR10':
model = models.CifarCnn()
elif dname == 'BreastCancerWisconsin':
model = models.BCNN()
elif dname in ('BankNote', 'HTRU2', 'Iris', 'WheatSeed'):
num_classes = dc.num_classes
num_features = dc.dim_data[0]
model = models.IrisNN(num_features=num_features,
hidden_nodes=num_features * 4,
num_classes=num_classes)
if model is None:
raise AttributeError('Cannot find model!')
modelname = model.__class__.__name__
logger.info('Selected %s model', modelname)
# set ModelContainer and train the model
mc = models.ModelContainerPT(model, dc)
mc.fit(max_epochs=max_epochs, batch_size=batch_size)
# save
if not os.path.exists('save'):
os.makedirs('save')
if filename is None:
filename = get_pt_model_filename(modelname, dname, max_epochs)
logger.debug('File name: %s', filename)
mc.save(filename, overwrite=overwrite)
# test result
file_path = os.path.join('save', filename)
logger.debug('Use saved parameters from %s', filename)
mc.load(file_path)
accuracy = mc.evaluate(dc.x_test, dc.y_test)
logger.info('Accuracy on test set: %f', accuracy)
