def generate_CIFAR10_adv(attacker_name, train_start=0, train_end=60000, test_start=0,
test_end=10000, nb_epochs=NB_EPOCHS, batch_size=BATCH_SIZE,
learning_rate=LEARNING_RATE,
clean_train=CLEAN_TRAIN,
testing=False,
nb_filters=NB_FILTERS, num_threads=None,
label_smoothing=0.1, args=FLAGS):
"""
CIFAR10 cleverhans tutorial
:param attacker_name:
:param train_start: index of first training set example
:param train_end: index of last training set example
:param test_start: index of first test set example
:param test_end: index of last test set example
:param nb_epochs: number of epochs to train model
:param batch_size: size of training batches
:param learning_rate: learning rate for training
:param clean_train: perform normal training on clean examples only
before performing adversarial training.
:param testing: if true, complete an AccuracyReport for unit tests
to verify that performance is adequate
:param label_smoothing: float, amount of label smoothing for cross entropy
:return: an AccuracyReport object
"""
if "batch_size" in ATTACK_PARAM[attacker_name]:
global BATCH_SIZE
batch_size = ATTACK_PARAM[attacker_name]["batch_size"]
BATCH_SIZE = batch_size
# Object used to keep track of (and return) key accuracies
report = AccuracyReport()
# Set TF random seed to improve reproducibility
tf.set_random_seed(1234)
# Set logging level to see debug information
set_log_level(logging.DEBUG)
# Create TF session
config_args = {}
if num_threads:
config_args = dict(intra_op_parallelism_threads=1)
config_args["gpu_options"] = tf.GPUOptions(allow_growth=True)
sess = tf.Session(config=tf.ConfigProto(**config_args))
# Get CIFAR10 data
data = CIFAR(train_start=train_start, train_end=train_end,
test_start=test_start, test_end=test_end)
dataset_size = data.x_train.shape[0]
dataset_train = data.to_tensorflow()[0]
dataset_train = dataset_train.map(
lambda x, y: (random_shift(random_horizontal_flip(x)), y), 4)
dataset_train = dataset_train.batch(batch_size)
dataset_train = dataset_train.prefetch(16)
x_train, y_train = data.get_set('train')
x_test, y_test = data.get_set('test')
# Use Image Parameters
img_rows, img_cols, nchannels = x_test.shape[1:4]
nb_classes = y_test.shape[1]
# Define input TF placeholder
x = tf.placeholder(tf.float32, shape=(BATCH_SIZE, img_rows, img_cols,
nchannels))
y = tf.placeholder(tf.float32, shape=(BATCH_SIZE, nb_classes))
# Train an MNIST model
train_params = {
'nb_epochs': nb_epochs,
'batch_size': batch_size,
'learning_rate': learning_rate
}
eval_params = {'batch_size': batch_size}
rng = np.random.RandomState([2017, 8, 30])
def do_generate_eval(adv_x, pred_adv_x, x_set, y_set, report_key, is_adv=None):
adv_images_total, adv_pred_total, gt_label_total, success_rate = untargeted_advx_image_eval(sess, x, y, adv_x,
pred_adv_x, x_set,
y_set,
args=eval_params)
setattr(report, report_key, success_rate)
if is_adv is None:
report_text = None
elif is_adv:
report_text = 'adversarial'
else:
report_text = 'legitimate'
if report_text:
print('adversarial attack successful rate on %s: %0.4f' % (report_text, success_rate))
return adv_images_total, adv_pred_total, gt_label_total, success_rate # shape = (total, H,W,C)
def do_eval(preds, x_set, y_set, report_key, is_adv=None):
acc = model_eval(sess, x, y, preds, x_set, y_set, args=eval_params)
setattr(report, report_key, acc)
if is_adv is None:
report_text = None
elif is_adv:
report_text = 'adversarial'
else:
report_text = 'legitimate'
if report_text:
print('Test accuracy on %s examples: %0.4f' % (report_text, acc))
if clean_train:
model = ModelAllConvolutional('model1', nb_classes, nb_filters,
input_shape=[32, 32, 3])
preds = model.get_logits(x) # tf.tensor
def evaluate():
do_eval(preds, x_test, y_test, 'clean_train_clean_eval', False)
resume_files = os.listdir(args.resume)
loss = CrossEntropy(model, smoothing=label_smoothing)
if len(resume_files) == 0:
saver = tf.train.Saver()
train(sess, loss, None, None,
dataset_train=dataset_train, dataset_size=dataset_size,
evaluate=evaluate, args=train_params, rng=rng,
var_list=model.get_params()) # 训练nb_epochs个epochs
save_path = saver.save(sess, "{}/model".format(args.resume), global_step=nb_epochs)
print("Model saved in path: %s" % save_path)
else:
# resume from old
latest_checkpoint = tf.train.latest_checkpoint(args.resume)
saver = tf.train.Saver()
saver.restore(sess, latest_checkpoint)
# Calculate training error
if testing:
evaluate()
# Initialize the Fast Gradient Sign Method (FGSM) attack object and
# graph
attacker = ATTACKERS[attacker_name](model, sess=sess)
param_dict = ATTACK_PARAM[attacker_name]
print("begin generate adversarial examples of CIFAR-10 using attacker: {}".format(attacker_name))
adv_x = attacker.generate(x, **param_dict) # tensor
preds_adv = model.get_logits(adv_x)
# generate adversarial examples
adv_images_total, adv_pred_total, gt_label_total, success_rate = do_generate_eval(adv_x, preds_adv, x_train,
y_train,
"clean_train_adv_eval", True)
print("attacker: {} attack successful rate for CIFAR-10 train dataset is {}".format(attacker_name, success_rate))
adv_images_total, adv_pred_total, gt_label_total, success_rate = do_generate_eval(adv_x, preds_adv, x_test,
y_test, "clean_test_adv_eval",
True)
print("attacker: {} attack successful rate for CIFAR-10 test dataset is {}".format(attacker_name, success_rate))
return report
def cifar10_tutorial(train_start=0,
train_end=60000,
test_start=0,
test_end=10000,
nb_epochs=NB_EPOCHS,
batch_size=BATCH_SIZE,
learning_rate=LEARNING_RATE,
clean_train=CLEAN_TRAIN,
testing=False,
backprop_through_attack=BACKPROP_THROUGH_ATTACK,
nb_filters=NB_FILTERS,
num_threads=None,
label_smoothing=0.1,
adversarial_training=ADVERSARIAL_TRAINING):
"""
CIFAR10 cleverhans tutorial
:param train_start: index of first training set example
:param train_end: index of last training set example
:param test_start: index of first test set example
:param test_end: index of last test set example
:param nb_epochs: number of epochs to train model
:param batch_size: size of training batches
:param learning_rate: learning rate for training
:param clean_train: perform normal training on clean examples only
before performing adversarial training.
:param testing: if true, complete an AccuracyReport for unit tests
to verify that performance is adequate
:param backprop_through_attack: If True, backprop through adversarial
example construction process during
adversarial training.
:param label_smoothing: float, amount of label smoothing for cross entropy
:param adversarial_training: True means using adversarial training
:return: an AccuracyReport object
"""
# Object used to keep track of (and return) key accuracies
report = AccuracyReport()
# Set TF random seed to improve reproducibility
tf.set_random_seed(1234)
# Set logging level to see debug information
set_log_level(logging.DEBUG)
# Create TF session
if num_threads:
config_args = dict(intra_op_parallelism_threads=1)
else:
# put data on cpu and gpu both
config_args = dict(allow_soft_placement=True)
sess = tf.Session(config=tf.ConfigProto(**config_args))
# Get CIFAR10 data
data = CIFAR10(train_start=train_start,
train_end=train_end,
test_start=test_start,
test_end=test_end)
dataset_size = data.x_train.shape[0]
dataset_train = data.to_tensorflow()[0]
dataset_train = dataset_train.map(
lambda x, y: (random_shift(random_horizontal_flip(x)), y), 4)
dataset_train = dataset_train.batch(batch_size)
dataset_train = dataset_train.prefetch(16)
x_train, y_train = data.get_set('train')
x_test, y_test = data.get_set('test')
# Use Image Parameters
img_rows, img_cols, nchannels = x_test.shape[1:4]
nb_classes = y_test.shape[1]
# Define input TF placeholder
x = tf.placeholder(tf.float32, shape=(None, img_rows, img_cols, nchannels))
y = tf.placeholder(tf.float32, shape=(None, nb_classes))
# Train an MNIST model
train_params = {
'nb_epochs': nb_epochs,
'batch_size': batch_size,
'learning_rate': learning_rate
}
eval_params = {'batch_size': batch_size}
bim_params = {
'eps': 0.5,
'clip_min': 0.,
'eps_iter': 0.002,
'nb_iter': 10,
'clip_max': 1.,
'ord': np.inf
}
rng = np.random.RandomState([2017, 8, 30])
def do_eval(preds, x_set, y_set, report_key, is_adv=None):
acc = model_eval(sess, x, y, preds, x_set, y_set, args=eval_params)
setattr(report, report_key, acc)
if is_adv is None:
report_text = None
elif is_adv:
report_text = 'adversarial'
else:
report_text = 'legitimate'
if report_text:
print('Test accuracy on %s examples: %0.4f' % (report_text, acc))
if clean_train:
model = ModelAllConvolutional('model1',
nb_classes,
nb_filters,
input_shape=[32, 32, 3])
preds = model.get_logits(x)
loss = CrossEntropy(model, smoothing=label_smoothing)
def evaluate():
do_eval(preds, x_test, y_test, 'clean_train_clean_eval', False)
"""
when training, evaluating can be happened
"""
train(sess,
loss,
None,
None,
dataset_train=dataset_train,
dataset_size=dataset_size,
evaluate=evaluate,
args=train_params,
rng=rng,
var_list=model.get_params())
# save model
# Calculate training error
if testing:
do_eval(preds, x_train, y_train, 'train_clean_train_clean_eval')
# Initialize the Basic Iterative Method (BIM) attack object and
# graph
for i in range(20):
bim = BasicIterativeMethod(model, sess=sess)
adv_x = bim.generate(x, **bim_params)
preds_adv = model.get_logits(adv_x)
# Evaluate the accuracy of the MNIST model on adversarial examples
print("eps:%0.2f" %
(bim_params["eps_iter"] * bim_params['nb_iter']))
do_eval(preds_adv, x_test, y_test, 'clean_train_adv_eval', True)
bim_params["eps_iter"] = bim_params["eps_iter"] + 0.002
# Calculate training error
if testing:
do_eval(preds_adv, x_train, y_train, 'train_clean_train_adv_eval')
if not adversarial_training:
return report
print('Repeating the process, using adversarial training')
# Create a new model and train it to be robust to BasicIterativeMethod
model2 = ModelAllConvolutional('model2',
nb_classes,
nb_filters,
input_shape=[32, 32, 3])
bim2 = BasicIterativeMethod(model2, sess=sess)
def attack(x):
return bim2.generate(x, **bim_params)
# add attack to loss
loss2 = CrossEntropy(model2, smoothing=label_smoothing, attack=attack)
preds2 = model2.get_logits(x)
adv_x2 = attack(x)
if not backprop_through_attack:
# For the fgsm attack used in this tutorial, the attack has zero
# gradient so enabling this flag does not change the gradient.
# For some other attacks, enabling this flag increases the cost of
# training, but gives the defender the ability to anticipate how
# the attacker will change their strategy in response to updates to
# the defender's parameters.
adv_x2 = tf.stop_gradient(adv_x2)
preds2_adv = model2.get_logits(adv_x2)
def evaluate2():
# Accuracy of adversarially trained model on legitimate test inputs
do_eval(preds2, x_test, y_test, 'adv_train_clean_eval', False)
# Accuracy of the adversarially trained model on adversarial examples
do_eval(preds2_adv, x_test, y_test, 'adv_train_adv_eval', True)
# Perform and evaluate adversarial training
train(sess,
loss2,
None,
None,
dataset_train=dataset_train,
dataset_size=dataset_size,
evaluate=evaluate2,
args=train_params,
rng=rng,
var_list=model2.get_params())
# Calculate training errors
if testing:
do_eval(preds2, x_train, y_train, 'train_adv_train_clean_eval')
do_eval(preds2_adv, x_train, y_train, 'train_adv_train_adv_eval')
return report
def cifar10_tutorial(
train_start=0,
train_end=60000,
test_start=0,
test_end=10000,
nb_epochs=NB_EPOCHS,
batch_size=BATCH_SIZE,
learning_rate=LEARNING_RATE,
clean_train=CLEAN_TRAIN,
testing=False,
backprop_through_attack=BACKPROP_THROUGH_ATTACK,
nb_filters=NB_FILTERS,
num_threads=None,
label_smoothing=0.1,
):
"""
CIFAR10 cleverhans tutorial
:param train_start: index of first training set example
:param train_end: index of last training set example
:param test_start: index of first test set example
:param test_end: index of last test set example
:param nb_epochs: number of epochs to train model
:param batch_size: size of training batches
:param learning_rate: learning rate for training
:param clean_train: perform normal training on clean examples only
before performing adversarial training.
:param testing: if true, complete an AccuracyReport for unit tests
to verify that performance is adequate
:param backprop_through_attack: If True, backprop through adversarial
example construction process during
adversarial training.
:param label_smoothing: float, amount of label smoothing for cross entropy
:return: an AccuracyReport object
"""
# Object used to keep track of (and return) key accuracies
report = AccuracyReport()
# Set TF random seed to improve reproducibility
tf.set_random_seed(1234)
# Set logging level to see debug information
set_log_level(logging.DEBUG)
# Create TF session
if num_threads:
config_args = dict(intra_op_parallelism_threads=1)
else:
config_args = {}
sess = tf.Session(config=tf.ConfigProto(**config_args))
# Get CIFAR10 data
data = CIFAR10(
train_start=train_start,
train_end=train_end,
test_start=test_start,
test_end=test_end,
)
dataset_size = data.x_train.shape[0]
dataset_train = data.to_tensorflow()[0]
dataset_train = dataset_train.map(
lambda x, y: (random_shift(random_horizontal_flip(x)), y), 4)
dataset_train = dataset_train.batch(batch_size)
dataset_train = dataset_train.prefetch(16)
x_train, y_train = data.get_set("train")
x_test, y_test = data.get_set("test")
# Use Image Parameters
img_rows, img_cols, nchannels = x_test.shape[1:4]
nb_classes = y_test.shape[1]
# Define input TF placeholder
x = tf.placeholder(tf.float32, shape=(None, img_rows, img_cols, nchannels))
y = tf.placeholder(tf.float32, shape=(None, nb_classes))
# Train an MNIST model
train_params = {
"nb_epochs": nb_epochs,
"batch_size": batch_size,
"learning_rate": learning_rate,
}
eval_params = {"batch_size": batch_size}
fgsm_params = {"eps": 0.3, "clip_min": 0.0, "clip_max": 1.0}
rng = np.random.RandomState([2017, 8, 30])
def do_eval(preds, x_set, y_set, report_key, is_adv=None):
acc = model_eval(sess, x, y, preds, x_set, y_set, args=eval_params)
setattr(report, report_key, acc)
if is_adv is None:
report_text = None
elif is_adv:
report_text = "adversarial"
else:
report_text = "legitimate"
if report_text:
print("Test accuracy on %s examples: %0.4f" % (report_text, acc))
if clean_train:
model = ModelAllConvolutional("model1",
nb_classes,
nb_filters,
input_shape=[32, 32, 3])
preds = model.get_logits(x)
loss = CrossEntropy(model, smoothing=label_smoothing)
def evaluate():
do_eval(preds, x_test, y_test, "clean_train_clean_eval", False)
train(
sess,
loss,
None,
None,
dataset_train=dataset_train,
dataset_size=dataset_size,
evaluate=evaluate,
args=train_params,
rng=rng,
var_list=model.get_params(),
)
# Calculate training error
if testing:
do_eval(preds, x_train, y_train, "train_clean_train_clean_eval")
# Initialize the Fast Gradient Sign Method (FGSM) attack object and
# graph
fgsm = FastGradientMethod(model, sess=sess)
adv_x = fgsm.generate(x, **fgsm_params)
preds_adv = model.get_logits(adv_x)
# Evaluate the accuracy of the MNIST model on adversarial examples
do_eval(preds_adv, x_test, y_test, "clean_train_adv_eval", True)
# Calculate training error
if testing:
do_eval(preds_adv, x_train, y_train, "train_clean_train_adv_eval")
print("Repeating the process, using adversarial training")
# Create a new model and train it to be robust to FastGradientMethod
model2 = ModelAllConvolutional("model2",
nb_classes,
nb_filters,
input_shape=[32, 32, 3])
fgsm2 = FastGradientMethod(model2, sess=sess)
def attack(x):
return fgsm2.generate(x, **fgsm_params)
loss2 = CrossEntropy(model2, smoothing=label_smoothing, attack=attack)
preds2 = model2.get_logits(x)
adv_x2 = attack(x)
if not backprop_through_attack:
# For the fgsm attack used in this tutorial, the attack has zero
# gradient so enabling this flag does not change the gradient.
# For some other attacks, enabling this flag increases the cost of
# training, but gives the defender the ability to anticipate how
# the atacker will change their strategy in response to updates to
# the defender's parameters.
adv_x2 = tf.stop_gradient(adv_x2)
preds2_adv = model2.get_logits(adv_x2)
def evaluate2():
# Accuracy of adversarially trained model on legitimate test inputs
do_eval(preds2, x_test, y_test, "adv_train_clean_eval", False)
# Accuracy of the adversarially trained model on adversarial examples
do_eval(preds2_adv, x_test, y_test, "adv_train_adv_eval", True)
# Perform and evaluate adversarial training
train(
sess,
loss2,
None,
None,
dataset_train=dataset_train,
dataset_size=dataset_size,
evaluate=evaluate2,
args=train_params,
rng=rng,
var_list=model2.get_params(),
)
# Calculate training errors
if testing:
do_eval(preds2, x_train, y_train, "train_adv_train_clean_eval")
do_eval(preds2_adv, x_train, y_train, "train_adv_train_adv_eval")
return report
def cifar10_tutorial(train_start=0,
train_end=60000,
test_start=0,
test_end=10000,
nb_epochs=NB_EPOCHS,
batch_size=BATCH_SIZE,
architecture=ARCHITECTURE,
load_model=LOAD_MODEL,
ckpt_dir='None',
learning_rate=LEARNING_RATE,
clean_train=CLEAN_TRAIN,
backprop_through_attack=BACKPROP_THROUGH_ATTACK,
nb_filters=NB_FILTERS,
num_threads=None,
label_smoothing=0.):
"""
CIFAR10 cleverhans tutorial
:param train_start: index of first training set example
:param train_end: index of last training set example
:param test_start: index of first test set example
:param test_end: index of last test set example
:param nb_epochs: number of epochs to train model
:param batch_size: size of training batches
:param learning_rate: learning rate for training
:param clean_train: perform normal training on clean examples only
before performing adversarial training.
:param backprop_through_attack: If True, backprop through adversarial
example construction process during
adversarial training.
:param label_smoothing: float, amount of label smoothing for cross entropy
:return: an AccuracyReport object
"""
# Object used to keep track of (and return) key accuracies
report = AccuracyReport()
# Set TF random seed to improve reproducibility
tf.set_random_seed(int(time.time() * 1000) % 2**31)
np.random.seed(int(time.time() * 1001) % 2**31)
# Set logging level to see debug information
set_log_level(logging.DEBUG)
# Create TF session
if num_threads:
config_args = dict(intra_op_parallelism_threads=1)
else:
config_args = {}
sess = tf.Session(config=tf.ConfigProto(**config_args))
# Get CIFAR10 data
data = CIFAR10(train_start=train_start,
train_end=train_end,
test_start=test_start,
test_end=test_end)
dataset_size = data.x_train.shape[0]
dataset_train = data.to_tensorflow()[0]
dataset_train = dataset_train.map(
lambda x, y: (random_shift(random_horizontal_flip(x)), y), 4)
dataset_train = dataset_train.batch(batch_size)
dataset_train = dataset_train.prefetch(16)
x_train, y_train = data.get_set('train')
pgd_train = None
if FLAGS.load_pgd_train_samples:
pgd_path = os.path.expanduser('~/data/advhyp/{}/samples'.format(
FLAGS.load_pgd_train_samples))
x_train = np.load(os.path.join(pgd_path, 'train_clean.npy'))
y_train = np.load(os.path.join(pgd_path, 'train_y.npy'))
pgd_train = np.load(os.path.join(pgd_path, 'train_pgd.npy'))
if x_train.shape[1] == 3:
x_train = x_train.transpose((0, 2, 3, 1))
pgd_train = pgd_train.transpose((0, 2, 3, 1))
if len(y_train.shape) == 1:
y_tmp = np.zeros((len(y_train), np.max(y_train) + 1),
y_train.dtype)
y_tmp[np.arange(len(y_tmp)), y_train] = 1.
y_train = y_tmp
x_test, y_test = data.get_set('test')
pgd_test = None
if FLAGS.load_pgd_test_samples:
pgd_path = os.path.expanduser('~/data/advhyp/{}/samples'.format(
FLAGS.load_pgd_test_samples))
x_test = np.load(os.path.join(pgd_path, 'test_clean.npy'))
y_test = np.load(os.path.join(pgd_path, 'test_y.npy'))
pgd_test = np.load(os.path.join(pgd_path, 'test_pgd.npy'))
if x_test.shape[1] == 3:
x_test = x_test.transpose((0, 2, 3, 1))
pgd_test = pgd_test.transpose((0, 2, 3, 1))
if len(y_test.shape) == 1:
y_tmp = np.zeros((len(y_test), np.max(y_test) + 1), y_test.dtype)
y_tmp[np.arange(len(y_tmp)), y_test] = 1.
y_test = y_tmp
train_idcs = np.arange(len(x_train))
np.random.shuffle(train_idcs)
x_train, y_train = x_train[train_idcs], y_train[train_idcs]
if pgd_train is not None:
pgd_train = pgd_train[train_idcs]
test_idcs = np.arange(len(x_test))[:FLAGS.test_size]
np.random.shuffle(test_idcs)
x_test, y_test = x_test[test_idcs], y_test[test_idcs]
if pgd_test is not None:
pgd_test = pgd_test[test_idcs]
# Use Image Parameters
img_rows, img_cols, nchannels = x_test.shape[1:4]
nb_classes = y_test.shape[1]
# Define input TF placeholder
x = tf.placeholder(tf.float32, shape=(None, img_rows, img_cols, nchannels))
y = tf.placeholder(tf.float32, shape=(None, nb_classes))
# Train an MNIST model
train_params = {
'nb_epochs': nb_epochs,
'batch_size': batch_size,
'learning_rate': learning_rate
}
eval_params = {'batch_size': batch_size}
pgd_params = {
# ord: ,
'eps': FLAGS.eps,
'eps_iter': (FLAGS.eps / 5),
'nb_iter': 10,
'clip_min': 0,
'clip_max': 255
}
cw_params = {
'binary_search_steps': FLAGS.cw_search_steps,
'max_iterations': FLAGS.cw_steps, #1000
'abort_early': True,
'learning_rate': FLAGS.cw_lr,
'batch_size': batch_size,
'confidence': 0,
'initial_const': FLAGS.cw_c,
'clip_min': 0,
'clip_max': 255
}
# Madry dosen't divide by 255
x_train *= 255
x_test *= 255
if pgd_train is not None:
pgd_train *= 255
if pgd_test is not None:
pgd_test *= 255
print('x_train amin={} amax={}'.format(np.amin(x_train), np.amax(x_train)))
print('x_test amin={} amax={}'.format(np.amin(x_test), np.amax(x_test)))
print(
'clip_min : {}, clip_max : {} >> CHECK WITH WHICH VALUES THE CLASSIFIER WAS PRETRAINED !!! <<'
.format(pgd_params['clip_min'], pgd_params['clip_max']))
rng = np.random.RandomState() # [2017, 8, 30]
debug_dict = dict() if FLAGS.save_debug_dict else None
def do_eval(preds,
x_set,
y_set,
report_key,
is_adv=None,
predictor=None,
x_adv=None):
if predictor is None:
acc = model_eval(sess, x, y, preds, x_set, y_set, args=eval_params)
else:
do_eval(preds, x_set, y_set, report_key, is_adv=is_adv)
if x_adv is not None:
x_set_adv, = batch_eval(sess, [x], [x_adv], [x_set],
batch_size=batch_size)
assert x_set.shape == x_set_adv.shape
x_set = x_set_adv
n_batches = math.ceil(x_set.shape[0] / batch_size)
p_set, p_det = np.concatenate([
predictor.send(x_set[b * batch_size:(b + 1) * batch_size])
for b in tqdm.trange(n_batches)
]).T
acc = np.equal(p_set, y_set[:len(p_set)].argmax(-1)).mean()
# if is_adv:
# import IPython ; IPython.embed() ; exit(1)
if FLAGS.save_debug_dict:
debug_dict['x_set'] = x_set
debug_dict['y_set'] = y_set
ddfn = 'logs/debug_dict_{}.pkl'.format(
'adv' if is_adv else 'clean')
if not os.path.exists(ddfn):
with open(ddfn, 'wb') as f:
pickle.dump(debug_dict, f)
debug_dict.clear()
if is_adv is None:
report_text = None
elif is_adv:
report_text = 'adversarial'
else:
report_text = 'legitimate'
if report_text:
print('Test accuracy on %s examples %s: %0.4f' %
(report_text, 'with correction'
if predictor is not None else 'without correction', acc))
if is_adv is not None:
label = 'test_acc_{}_{}'.format(
report_text, 'corrected' if predictor else 'uncorrected')
swriter.add_scalar(label, acc)
if predictor is not None:
detect = np.equal(p_det, is_adv).mean()
label = 'test_det_{}_{}'.format(
report_text,
'corrected' if predictor else 'uncorrected')
print(label, detect)
swriter.add_scalar(label, detect)
label = 'test_dac_{}_{}'.format(
report_text,
'corrected' if predictor else 'uncorrected')
swriter.add_scalar(
label,
np.equal(p_set,
y_set[:len(p_set)].argmax(-1))[np.equal(
p_det, is_adv)].mean())
return acc
if clean_train:
if architecture == 'ConvNet':
model = ModelAllConvolutional('model1',
nb_classes,
nb_filters,
input_shape=[32, 32, 3])
elif architecture == 'ResNet':
model = ResNet(scope='ResNet')
else:
raise Exception('Specify valid classifier architecture!')
preds = model.get_logits(x)
loss = CrossEntropy(model, smoothing=label_smoothing)
if load_model:
model_name = 'naturally_trained'
if FLAGS.load_adv_trained:
model_name = 'adv_trained'
if ckpt_dir is not 'None':
ckpt = tf.train.get_checkpoint_state(
os.path.join(os.path.expanduser(ckpt_dir), model_name))
else:
ckpt = tf.train.get_checkpoint_state('./models/' + model_name)
ckpt_path = False if ckpt is None else ckpt.model_checkpoint_path
saver = tf.train.Saver(var_list=dict(
(v.name.split('/', 1)[1].split(':')[0], v)
for v in tf.global_variables()))
saver.restore(sess, ckpt_path)
print('\nMODEL SUCCESSFULLY LOADED from : {}'.format(ckpt_path))
initialize_uninitialized_global_variables(sess)
else:
def evaluate():
do_eval(preds, x_test, y_test, 'clean_train_clean_eval', False)
train(sess,
loss,
None,
None,
dataset_train=dataset_train,
dataset_size=dataset_size,
evaluate=evaluate,
args=train_params,
rng=rng,
var_list=model.get_params())
logits_op = preds.op
while logits_op.type != 'MatMul':
logits_op = logits_op.inputs[0].op
latent_x_tensor, weights = logits_op.inputs
logits_tensor = preds
nb_classes = weights.shape[-1].value
if not FLAGS.save_pgd_samples:
noise_eps = FLAGS.noise_eps.split(',')
if FLAGS.noise_eps_detect is None:
FLAGS.noise_eps_detect = FLAGS.noise_eps
noise_eps_detect = FLAGS.noise_eps_detect.split(',')
if pgd_train is not None:
pgd_train = pgd_train[:FLAGS.n_collect]
if not FLAGS.passthrough:
predictor = tf_robustify.collect_statistics(
x_train[:FLAGS.n_collect],
y_train[:FLAGS.n_collect],
x,
sess,
logits_tensor=logits_tensor,
latent_x_tensor=latent_x_tensor,
weights=weights,
nb_classes=nb_classes,
p_ratio_cutoff=FLAGS.p_ratio_cutoff,
noise_eps=noise_eps,
noise_eps_detect=noise_eps_detect,
pgd_eps=pgd_params['eps'],
pgd_lr=pgd_params['eps_iter'] / pgd_params['eps'],
pgd_iters=pgd_params['nb_iter'],
save_alignments_dir='logs/stats'
if FLAGS.save_alignments else None,
load_alignments_dir=os.path.expanduser(
'~/data/advhyp/madry/stats')
if FLAGS.load_alignments else None,
clip_min=pgd_params['clip_min'],
clip_max=pgd_params['clip_max'],
batch_size=batch_size,
num_noise_samples=FLAGS.num_noise_samples,
debug_dict=debug_dict,
debug=FLAGS.debug,
targeted=False,
pgd_train=pgd_train,
fit_classifier=FLAGS.fit_classifier,
clip_alignments=FLAGS.clip_alignments,
just_detect=FLAGS.just_detect)
else:
def _predictor():
_x = yield
while (_x is not None):
_y = sess.run(preds, {x: _x}).argmax(-1)
_x = yield np.stack((_y, np.zeros_like(_y)), -1)
predictor = _predictor()
next(predictor)
if FLAGS.save_alignments:
exit(0)
# Evaluate the accuracy of the model on clean examples
acc_clean = do_eval(preds,
x_test,
y_test,
'clean_train_clean_eval',
False,
predictor=predictor)
# Initialize the PGD attack object and graph
if FLAGS.attack == 'pgd':
pgd = MadryEtAl(model, sess=sess)
adv_x = pgd.generate(x, **pgd_params)
elif FLAGS.attack == 'cw':
cw = CarliniWagnerL2(model, sess=sess)
adv_x = cw.generate(x, **cw_params)
elif FLAGS.attack == 'mean':
pgd = MadryEtAl(model, sess=sess)
mean_eps = FLAGS.mean_eps * FLAGS.eps
def _attack_mean(x):
x_many = tf.tile(x[None], (FLAGS.mean_samples, 1, 1, 1))
x_noisy = x_many + tf.random_uniform(x_many.shape, -mean_eps,
mean_eps)
x_noisy = tf.clip_by_value(x_noisy, 0, 255)
x_pgd = pgd.generate(x_noisy, **pgd_params)
x_clip = tf.minimum(x_pgd, x_many + FLAGS.eps)
x_clip = tf.maximum(x_clip, x_many - FLAGS.eps)
x_clip = tf.clip_by_value(x_clip, 0, 255)
return x_clip
adv_x = tf.map_fn(_attack_mean, x)
adv_x = tf.reduce_mean(adv_x, 1)
preds_adv = model.get_logits(adv_x)
if FLAGS.save_pgd_samples:
for ds, y, name in ((x_train, y_train, 'train'), (x_test, y_test,
'test')):
train_batches = math.ceil(len(ds) / FLAGS.batch_size)
train_pgd = np.concatenate([
sess.run(adv_x, {
x:
ds[b * FLAGS.batch_size:(b + 1) * FLAGS.batch_size]
}) for b in tqdm.trange(train_batches)
])
np.save('logs/{}_clean.npy'.format(name), ds / 255.)
np.save('logs/{}_y.npy'.format(name), y)
train_pgd /= 255.
np.save('logs/{}_pgd.npy'.format(name), train_pgd)
exit(0)
# Evaluate the accuracy of the model on adversarial examples
if not FLAGS.load_pgd_test_samples:
acc_pgd = do_eval(preds_adv,
x_test,
y_test,
'clean_train_adv_eval',
True,
predictor=predictor,
x_adv=adv_x)
else:
acc_pgd = do_eval(preds,
pgd_test,
y_test,
'clean_train_adv_eval',
True,
predictor=predictor)
swriter.add_scalar('test_acc_mean', (acc_clean + acc_pgd) / 2., 0)
print('Repeating the process, using adversarial training')
exit(0)
# Create a new model and train it to be robust to MadryEtAl
if architecture == 'ConvNet':
model2 = ModelAllConvolutional('model2',
nb_classes,
nb_filters,
input_shape=[32, 32, 3])
elif architecture == 'ResNet':
model = ResNet()
else:
raise Exception('Specify valid classifier architecture!')
pgd2 = MadryEtAl(model2, sess=sess)
def attack(x):
return pgd2.generate(x, **pgd_params)
loss2 = CrossEntropy(model2, smoothing=label_smoothing, attack=attack)
preds2 = model2.get_logits(x)
adv_x2 = attack(x)
if not backprop_through_attack:
# For some attacks, enabling this flag increases the cost of
# training, but gives the defender the ability to anticipate how
# the atacker will change their strategy in response to updates to
# the defender's parameters.
adv_x2 = tf.stop_gradient(adv_x2)
preds2_adv = model2.get_logits(adv_x2)
if load_model:
if ckpt_dir is not 'None':
ckpt = tf.train.get_checkpoint_state(
os.path.join(os.path.expanduser(ckpt_dir), 'adv_trained'))
else:
ckpt = tf.train.get_checkpoint_state('./models/adv_trained')
ckpt_path = False if ckpt is None else ckpt.model_checkpoint_path
assert ckpt_path and tf_model_load(
sess, file_path=ckpt_path), '\nMODEL LOADING FAILED'
print('\nMODEL SUCCESSFULLY LOADED from : {}'.format(ckpt_path))
initialize_uninitialized_global_variables(sess)
else:
def evaluate2():
# Accuracy of adversarially trained model on legitimate test inputs
do_eval(preds2, x_test, y_test, 'adv_train_clean_eval', False)
# Accuracy of the adversarially trained model on adversarial
# examples
do_eval(preds2_adv, x_test, y_test, 'adv_train_adv_eval', True)
# Perform and evaluate adversarial training
train(sess,
loss2,
None,
None,
dataset_train=dataset_train,
dataset_size=dataset_size,
evaluate=evaluate2,
args=train_params,
rng=rng,
var_list=model2.get_params())
# Evaluate model
do_eval(preds2, x_test, y_test, 'adv_train_clean_eval', False)
do_eval(preds2_adv, x_test, y_test, 'adv_train_adv_eval', True)
return report
def cifar10_tutorial_bim(train_start=0,
train_end=60000,
test_start=0,
test_end=10000,
viz_enabled=VIZ_ENABLED,
nb_epochs=NB_EPOCHS,
batch_size=BATCH_SIZE,
source_samples=SOURCE_SAMPLES,
learning_rate=LEARNING_RATE,
attack_iterations=ATTACK_ITERATIONS,
model_path=MODEL_PATH,
targeted=TARGETED,
noise_output=NOISE_OUTPUT):
"""
CIFAR10 tutorial for Basic Iterative Method's attack
:param train_start: index of first training set example
:param train_end: index of last training set example
:param test_start: index of first test set example
:param test_end: index of last test set example
:param viz_enabled: (boolean) activate plots of adversarial examples
:param nb_epochs: number of epochs to train model
:param batch_size: size of training batches
:param nb_classes: number of output classes
:param source_samples: number of test inputs to attack
:param learning_rate: learning rate for training
:param model_path: path to the model file
:param targeted: should we run a targeted attack? or untargeted?
:return: an AccuracyReport object
"""
# Object used to keep track of (and return) key accuracies
report = AccuracyReport()
# Set TF random seed to improve reproducibility
tf.set_random_seed(1234)
# Create TF session
sess = tf.Session()
print("Created TensorFlow session.")
set_log_level(logging.DEBUG)
# Get CIFAR10 test data
cifar10 = CIFAR10(train_start=train_start,
train_end=train_end,
test_start=test_start,
test_end=test_end)
x_train, y_train = cifar10.get_set('train')
x_test, y_test = cifar10.get_set('test')
# Obtain Image Parameters
img_rows, img_cols, nchannels = x_train.shape[1:4]
nb_classes = y_train.shape[1]
# Define input TF placeholder
x = tf.placeholder(tf.float32, shape=(None, img_rows, img_cols, nchannels))
y = tf.placeholder(tf.float32, shape=(None, nb_classes))
nb_filters = 64
# Define TF model graph
model = ModelAllConvolutional('model1',
nb_classes,
nb_filters,
input_shape=[32, 32, 3])
preds = model.get_logits(x)
loss = CrossEntropy(model, smoothing=0.1)
print("Defined TensorFlow model graph.")
###########################################################################
# Training the model using TensorFlow
###########################################################################
# Train an CIFAR10 model
train_params = {
'nb_epochs': nb_epochs,
'batch_size': batch_size,
'learning_rate': learning_rate,
'filename': os.path.split(model_path)[-1]
}
rng = np.random.RandomState([2017, 8, 30])
# check if we've trained before, and if we have, use that pre-trained model
if os.path.exists(model_path + ".meta"):
tf_model_load(sess, model_path)
else:
train(sess, loss, x_train, y_train, args=train_params, rng=rng)
saver = tf.train.Saver()
saver.save(sess, model_path)
# Evaluate the accuracy of the CIFAR10 model on legitimate test examples
eval_params = {'batch_size': batch_size}
accuracy = model_eval(sess, x, y, preds, x_test, y_test, args=eval_params)
assert x_test.shape[0] == test_end - test_start, x_test.shape
print('Test accuracy on legitimate test examples: {0}'.format(accuracy))
report.clean_train_clean_eval = accuracy
###########################################################################
# Craft adversarial examples using Basic Iterative Method's approach
###########################################################################
nb_adv_per_sample = str(nb_classes - 1) if targeted else '1'
print('Crafting ' + str(source_samples) + ' * ' + nb_adv_per_sample +
' adversarial examples')
print("This could take some time ...")
# Instantiate a BIM attack object
bim = BasicIterativeMethod(model, sess=sess)
if viz_enabled:
assert source_samples == nb_classes
idxs = [
np.where(np.argmax(y_test, axis=1) == i)[0][0]
for i in range(nb_classes)
]
if targeted:
if viz_enabled:
# Initialize our array for grid visualization
grid_shape = (nb_classes, 1, img_rows, img_cols, nchannels)
grid_viz_data = np.zeros(grid_shape, dtype='f')
adv_inputs = np.array([[instance] * nb_classes
for instance in x_test[idxs]],
dtype=np.float32)
else:
adv_inputs = np.array([[instance] * nb_classes
for instance in x_test[:source_samples]],
dtype=np.float32)
one_hot = np.zeros((nb_classes, nb_classes))
one_hot[np.arange(nb_classes), np.arange(nb_classes)] = 1
adv_inputs = adv_inputs.reshape(
(source_samples * nb_classes, img_rows, img_cols, nchannels))
adv_ys = np.array([one_hot] * source_samples,
dtype=np.float32).reshape(
(source_samples * nb_classes, nb_classes))
else:
if viz_enabled:
# Initialize our array for grid visualization
grid_shape = (nb_classes, nb_classes, img_rows, img_cols,
nchannels)
grid_viz_data = np.zeros(grid_shape, dtype='f')
adv_inputs = x_test[idxs]
else:
adv_inputs = x_test[:source_samples]
adv_ys = None
bim_params = {
'eps': 0.3,
'clip_min': 0.,
'clip_max': 1.,
'nb_iter': 50,
'eps_iter': .01
}
adv = bim.generate_np(adv_inputs, **bim_params)
eval_params = {'batch_size': np.minimum(nb_classes, source_samples)}
if targeted:
adv_accuracy = model_eval(sess,
x,
y,
preds,
adv,
adv_ys,
args=eval_params)
else:
if viz_enabled:
err = model_eval(sess,
x,
y,
preds,
adv,
y_test[idxs],
args=eval_params)
adv_accuracy = 1 - err
else:
err = model_eval(sess,
x,
y,
preds,
adv,
y_test[:source_samples],
args=eval_params)
adv_accuracy = 1 - err
if viz_enabled:
for i in range(nb_classes):
if noise_output:
image = adv[i * nb_classes] - adv_inputs[i * nb_classes]
else:
image = adv[i * nb_classes]
grid_viz_data[i, 0] = image
print('--------------------------------------')
# Compute the number of adversarial examples that were successfully found
print('Avg. rate of successful adv. examples {0:.4f}'.format(adv_accuracy))
report.clean_train_adv_eval = 1. - adv_accuracy
# Compute the average distortion introduced by the algorithm
percent_perturbed = np.mean(
np.sum((adv - adv_inputs)**2, axis=(1, 2, 3))**.5)
print('Avg. L_2 norm of perturbations {0:.4f}'.format(percent_perturbed))
# Close TF session
sess.close()
def save_visual(data, path):
"""
Modified version of cleverhans.plot.pyplot
"""
figure = plt.figure()
# figure.canvas.set_window_title('Cleverhans: Grid Visualization')
# Add the images to the plot
num_cols = data.shape[0]
num_rows = data.shape[1]
num_channels = data.shape[4]
for y in range(num_rows):
for x in range(num_cols):
figure.add_subplot(num_rows, num_cols,
(x + 1) + (y * num_cols))
plt.axis('off')
if num_channels == 1:
plt.imshow(data[x, y, :, :, 0], cmap='gray')
else:
plt.imshow(data[x, y, :, :, :])
# Draw the plot and return
plt.savefig(path)
return figure
# Finally, block & display a grid of all the adversarial examples
if viz_enabled:
if noise_output:
image_name = "output/bim_cifar10_noise.png"
else:
image_name = "output/bim_cifar10.png"
_ = save_visual(grid_viz_data, image_name)
return report
def cifar10_tutorial(train_start=0,
train_end=60000,
test_start=0,
test_end=10000,
nb_epochs=NB_EPOCHS,
batch_size=BATCH_SIZE,
learning_rate=LEARNING_RATE,
clean_train=CLEAN_TRAIN,
testing=False,
backprop_through_attack=BACKPROP_THROUGH_ATTACK,
nb_filters=NB_FILTERS,
num_threads=None,
label_smoothing=0.1,
retrain=False):
"""
CIFAR10 cleverhans tutorial
:param train_start: index of first training set example
:param train_end: index of last training set example
:param test_start: index of first test set example
:param test_end: index of last test set example
:param nb_epochs: number of epochs to train model
:param batch_size: size of training batches
:param learning_rate: learning rate for training
:param clean_train: perform normal training on clean examples only
before performing adversarial training.
:param testing: if true, complete an AccuracyReport for unit tests
to verify that performance is adequate
:param backprop_through_attack: If True, backprop through adversarial
example construction process during
adversarial training.
:param label_smoothing: float, amount of label smoothing for cross entropy
:return: an AccuracyReport object
"""
# Object used to keep track of (and return) key accuracies
report = AccuracyReport()
# Set TF random seed to improve reproducibility
tf.set_random_seed(1234)
# Set logging level to see debug information
set_log_level(logging.DEBUG)
# Create TF session
if num_threads:
config_args = dict(intra_op_parallelism_threads=1)
else:
config_args = {}
sess = tf.Session(config=tf.ConfigProto(**config_args))
# Get CIFAR10 data
data = CIFAR10(train_start=train_start,
train_end=train_end,
test_start=test_start,
test_end=test_end)
dataset_size = data.x_train.shape[0]
dataset_train = data.to_tensorflow()[0]
dataset_train = dataset_train.map(
lambda x, y: (random_shift(random_horizontal_flip(x)), y), 4)
dataset_train = dataset_train.batch(batch_size)
dataset_train = dataset_train.prefetch(16)
x_train, y_train = data.get_set('train')
x_test, y_test = data.get_set('test')
# start = 6
# end = 10
# x_test = x_test[start:end]
# y_test = y_test[start:end]
###########################
# Adjust hue / saturation #
###########################
# hueValue = 0.9
# saturationValue = 0.9
# tf_x_test = tf.image.adjust_saturation(tf.image.adjust_hue(x_test, saturationValue), hueValue)
# tf_x_test = tf.image.adjust_saturation(tx_test, hueValue)
# x_test = sess.run(tf_x_test)
###############################
# Transform image to uniimage #
###############################
# x_train = convert_uniimage(x_train)
# Use Image Parameters
img_rows, img_cols, nchannels = x_test.shape[1:4]
nb_classes = y_test.shape[1]
# Define input TF placeholder
x = tf.placeholder(tf.float32, shape=(None, img_rows, img_cols, nchannels))
y = tf.placeholder(tf.float32, shape=(None, nb_classes))
# Train an MNIST model
train_params = {
'nb_epochs': nb_epochs,
'batch_size': batch_size,
'learning_rate': learning_rate,
'train_dir': save_dir,
'filename': filename,
}
eval_params = {'batch_size': batch_size}
fgsm_params = {'eps': 8 / 255, 'clip_min': 0., 'clip_max': 1.}
rng = np.random.RandomState([2017, 8, 30])
def do_eval(preds,
x_set,
y_set,
report_key,
is_adv=None,
ae=None,
type=None,
datasetName=None,
discretizeColor=1):
accuracy, distortion = model_eval(sess,
x,
y,
preds,
x_set,
y_set,
args=eval_params,
is_adv=is_adv,
ae=ae,
type=type,
datasetName=datasetName,
discretizeColor=discretizeColor)
setattr(report, report_key, accuracy)
if is_adv is None:
report_text = None
elif is_adv:
report_text = 'adversarial'
else:
report_text = 'legitimate'
if report_text:
print('Test accuracy on %s examples: %0.4f' %
(report_text, accuracy))
return accuracy, distortion
if clean_train:
model = ModelAllConvolutional('model1',
nb_classes,
nb_filters,
input_shape=[32, 32, 3])
# model = UIPModel('model1', nb_classes, nb_filters, input_shape=[32, 32, 3])
preds = model.get_logits(x)
loss = CrossEntropy(model, smoothing=label_smoothing)
def evaluate():
do_eval(preds,
x_test,
y_test,
'clean_train_clean_eval',
False,
type=type,
datasetName="CIFAR10",
discretizeColor=discretizeColor)
# train(sess, loss, None, None,
# dataset_train=dataset_train, dataset_size=dataset_size,
# evaluate=evaluate, args=train_params, rng=rng,
# var_list=model.get_params(), save=save)
saveFileNumArr = []
# saveFileNumArr = [50, 500, 1000]
count = 0
appendNum = 1000
while count < 1000:
count = count + appendNum
saveFileNumArr.append(count)
distortionArr = []
accuracyArr = []
for i in range(len(saveFileNumArr)):
saveFileNum = saveFileNumArr[i]
model_path = os.path.join(save_dir,
filename + "-" + str(saveFileNum))
print("Trying to load trained model from: " + model_path)
if os.path.exists(model_path + ".meta"):
tf_model_load(sess, model_path)
print("Load trained model")
else:
train_with_noise(sess,
loss,
x_train,
y_train,
evaluate=evaluate,
args=train_params,
rng=rng,
var_list=model.get_params(),
save=save,
type=type,
datasetName="CIFAR10",
retrain=retrain,
discretizeColor=discretizeColor)
retrain = False
##########################################
# Generate semantic adversarial examples #
##########################################
adv_x, y_test2 = color_shift_attack(sess,
x,
y,
np.copy(x_test),
np.copy(y_test),
preds,
args=eval_params,
num_trials=num_trials)
x_test2 = adv_x
# convert_uniimage(np.copy(x_test2), np.copy(x_test), discretizeColor)
accuracy, distortion = do_eval(preds,
np.copy(x_test2),
np.copy(y_test2),
'clean_train_clean_eval',
False,
type=type,
datasetName="CIFAR10",
discretizeColor=discretizeColor)
# accuracy, distortion = do_eval(preds, np.copy(x_test), np.copy(y_test), 'clean_train_clean_eval', False, type=type,
# datasetName="CIFAR10", discretizeColor=discretizeColor)
# # Initialize the Fast Gradient Sign Method (FGSM) attack object and
# # graph
# fgsm = FastGradientMethod(model, sess=sess)
# fgsm = BasicIterativeMethod(model, sess=sess)
# fgsm = MomentumIterativeMethod(model, sess=sess)
# adv_x = fgsm.generate(x, **fgsm_params)
# preds_adv = model.get_logits(adv_x)
# Evaluate the accuracy of the MNIST model on adversarial examples
# accuracy, distortion = do_eval(preds_adv, x_test, y_test, 'clean_train_adv_eval', True, type=type)
# accuracy, distortion = do_eval(preds, x_test, y_test, 'clean_train_adv_eval', True, ae=adv_x, type=type,
# datasetName="CIFAR10", discretizeColor=discretizeColor)
distortionArr.append(distortion)
accuracyArr.append(accuracy)
print(str(accuracy))
print(str(distortion))
print("accuracy:")
for accuracy in accuracyArr:
print(accuracy)
print("distortion:")
for distortion in distortionArr:
print(distortion)
# print("hue "+str(hueValue))
return report
def cifar10_tutorial_jsma(train_start=0,
train_end=60000,
test_start=0,
test_end=10000,
viz_enabled=VIZ_ENABLED,
nb_epochs=NB_EPOCHS,
batch_size=BATCH_SIZE,
source_samples=SOURCE_SAMPLES,
learning_rate=LEARNING_RATE,
model_path=MODEL_PATH,
noise_output=NOISE_OUTPUT):
"""
CIFAR10 tutorial for the Jacobian-based saliency map approach (JSMA)
:param train_start: index of first training set example
:param train_end: index of last training set example
:param test_start: index of first test set example
:param test_end: index of last test set example
:param viz_enabled: (boolean) activate plots of adversarial examples
:param nb_epochs: number of epochs to train model
:param batch_size: size of training batches
:param nb_classes: number of output classes
:param source_samples: number of test inputs to attack
:param learning_rate: learning rate for training
:return: an AccuracyReport object
"""
# Object used to keep track of (and return) key accuracies
report = AccuracyReport()
# Set TF random seed to improve reproducibility
tf.set_random_seed(1234)
# Create TF session and set as Keras backend session
sess = tf.Session()
print("Created TensorFlow session.")
set_log_level(logging.DEBUG)
# Get CIFAR10 test data
cifar10 = CIFAR10(train_start=train_start,
train_end=train_end,
test_start=test_start,
test_end=test_end)
x_train, y_train = cifar10.get_set('train')
x_test, y_test = cifar10.get_set('test')
# Obtain Image Parameters
img_rows, img_cols, nchannels = x_train.shape[1:4]
nb_classes = y_train.shape[1]
# Define input TF placeholder
x = tf.placeholder(tf.float32, shape=(None, img_rows, img_cols, nchannels))
y = tf.placeholder(tf.float32, shape=(None, nb_classes))
nb_filters = 64
# Define TF model graph
model = ModelAllConvolutional('model1',
nb_classes,
nb_filters,
input_shape=[32, 32, 3])
preds = model.get_logits(x)
loss = CrossEntropy(model, smoothing=0.1)
print("Defined TensorFlow model graph.")
###########################################################################
# Training the model using TensorFlow
###########################################################################
# Train an CIFAR10 model
train_params = {
'nb_epochs': nb_epochs,
'batch_size': batch_size,
'learning_rate': learning_rate,
'filename': os.path.split(model_path)[-1]
}
sess.run(tf.global_variables_initializer())
rng = np.random.RandomState([2017, 8, 30])
train(sess, loss, x_train, y_train, args=train_params, rng=rng)
# Evaluate the accuracy of the CIFAR10 model on legitimate test examples
eval_params = {'batch_size': batch_size}
accuracy = model_eval(sess, x, y, preds, x_test, y_test, args=eval_params)
assert x_test.shape[0] == test_end - test_start, x_test.shape
print('Test accuracy on legitimate test examples: {0}'.format(accuracy))
report.clean_train_clean_eval = accuracy
###########################################################################
# Craft adversarial examples using the Jacobian-based saliency map approach
###########################################################################
print('Crafting ' + str(source_samples) + ' * ' + str(nb_classes - 1) +
' adversarial examples')
# Keep track of success (adversarial example classified in target)
results = np.zeros((nb_classes, source_samples), dtype='i')
# Rate of perturbed features for each test set example and target class
perturbations = np.zeros((nb_classes, source_samples), dtype='f')
# Initialize our array for grid visualization
grid_shape = (nb_classes, 1, img_rows, img_cols, nchannels)
grid_viz_data = np.zeros(grid_shape, dtype='f')
# Instantiate a SaliencyMapMethod attack object
jsma = SaliencyMapMethod(model, sess=sess)
jsma_params = {
'theta': 1.,
'gamma': 0.1,
'clip_min': 0.,
'clip_max': 1.,
'y_target': None
}
# Loop over the samples we want to perturb into adversarial examples
adv_all = np.zeros((nb_classes, img_rows, img_cols, nchannels), dtype='f')
sample_all = np.zeros((nb_classes, img_rows, img_cols, nchannels),
dtype='f')
for sample_ind in xrange(0, source_samples):
print('--------------------------------------')
print('Attacking input %i/%i' % (sample_ind + 1, source_samples))
sample = x_test[sample_ind:(sample_ind + 1)]
# We want to find an adversarial example for each possible target class
# (i.e. all classes that differ from the label given in the dataset)
current_class = int(np.argmax(y_test[sample_ind]))
target_classes = other_classes(nb_classes, current_class)
# For the grid visualization, keep original images along the diagonal
# grid_viz_data[current_class, current_class, :, :, :] = np.reshape(
# sample, (img_rows, img_cols, nchannels))
# Loop over all target classes
for target in target_classes:
print('Generating adv. example for target class %i' % target)
# This call runs the Jacobian-based saliency map approach
one_hot_target = np.zeros((1, nb_classes), dtype=np.float32)
one_hot_target[0, target] = 1
jsma_params['y_target'] = one_hot_target
adv_x = jsma.generate_np(sample, **jsma_params)
adv_all[current_class] = adv_x
sample_all[current_class] = sample
# Check if success was achieved
res = int(model_argmax(sess, x, preds, adv_x) == target)
# Computer number of modified features
adv_x_reshape = adv_x.reshape(-1)
test_in_reshape = x_test[sample_ind].reshape(-1)
nb_changed = np.where(adv_x_reshape != test_in_reshape)[0].shape[0]
percent_perturb = float(nb_changed) / adv_x.reshape(-1).shape[0]
# Display the original and adversarial images side-by-side
# if viz_enabled:
# figure = pair_visual(
# np.reshape(sample, (img_rows, img_cols, nchannels)),
# np.reshape(adv_x, (img_rows, img_cols, nchannels)), figure)
# # Add our adversarial example to our grid data
# grid_viz_data[target, current_class, :, :, :] = np.reshape(
# adv_x, (img_rows, img_cols, nchannels))
# Update the arrays for later analysis
results[target, sample_ind] = res
perturbations[target, sample_ind] = percent_perturb
print('--------------------------------------')
# Compute the number of adversarial examples that were successfully found
nb_targets_tried = ((nb_classes - 1) * source_samples)
succ_rate = float(np.sum(results)) / nb_targets_tried
print('Avg. rate of successful adv. examples {0:.4f}'.format(succ_rate))
report.clean_train_adv_eval = 1. - succ_rate
# Compute the average distortion introduced by the algorithm
percent_perturbed = np.mean(perturbations)
print('Avg. rate of perturbed features {0:.4f}'.format(percent_perturbed))
# Compute the average distortion introduced for successful samples only
percent_perturb_succ = np.mean(perturbations * (results == 1))
print('Avg. rate of perturbed features for successful '
'adversarial examples {0:.4f}'.format(percent_perturb_succ))
# Compute the average distortion introduced by the algorithm
l2_norm = np.mean(np.sum((adv_all - sample_all)**2, axis=(1, 2, 3))**.5)
print('Avg. L_2 norm of perturbations {0:.4f}'.format(l2_norm))
for i in range(nb_classes):
if noise_output:
image = adv_all[i] - sample_all[i]
else:
image = adv_all[i]
grid_viz_data[i, 0] = image
# Close TF session
sess.close()
def save_visual(data, path):
"""
Modified version of cleverhans.plot.pyplot
"""
import matplotlib.pyplot as plt
figure = plt.figure()
# figure.canvas.set_window_title('Cleverhans: Grid Visualization')
# Add the images to the plot
num_cols = data.shape[0]
num_rows = data.shape[1]
num_channels = data.shape[4]
for y in range(num_rows):
for x in range(num_cols):
figure.add_subplot(num_rows, num_cols,
(x + 1) + (y * num_cols))
plt.axis('off')
if num_channels == 1:
plt.imshow(data[x, y, :, :, 0], cmap='gray')
else:
plt.imshow(data[x, y, :, :, :])
# Draw the plot and return
plt.savefig(path)
# Finally, block & display a grid of all the adversarial examples
if viz_enabled:
if noise_output:
image_name = "output/jsma_cifar10_noise.png"
else:
image_name = "output/jsma_cifar10.png"
_ = save_visual(grid_viz_data, image_name)
return report
def cifar10_tutorial(train_start=0,
train_end=60000,
test_start=0,
test_end=10000,
nb_epochs=NB_EPOCHS,
batch_size=BATCH_SIZE,
learning_rate=LEARNING_RATE,
train_dir=TRAIN_DIR,
filename=FILENAME,
load_model=LOAD_MODEL,
testing=False,
label_smoothing=0.1):
"""
MNIST CleverHans tutorial
:param train_start: index of first training set example
:param train_end: index of last training set example
:param test_start: index of first test set example
:param test_end: index of last test set example
:param nb_epochs: number of epochs to train model
:param batch_size: size of training batches
:param learning_rate: learning rate for training
:param train_dir: Directory storing the saved model
:param filename: Filename to save model under
:param load_model: True for load, False for not load
:param testing: if true, test error is calculated
:param label_smoothing: float, amount of label smoothing for cross entropy
:return: an AccuracyReport object
"""
keras.layers.core.K.set_learning_phase(0)
# Object used to keep track of (and return) key accuracies
report = AccuracyReport()
# Set TF random seed to improve reproducibility
tf.set_random_seed(1234)
if not hasattr(backend, "tf"):
raise RuntimeError("This tutorial requires keras to be configured"
" to use the TensorFlow backend.")
if keras.backend.image_dim_ordering() != 'tf':
keras.backend.set_image_dim_ordering('tf')
print("INFO: '~/.keras/keras.json' sets 'image_dim_ordering' to "
"'th', temporarily setting to 'tf'")
# Create TF session and set as Keras backend session
os.environ["CUDA_VISIBLE_DEVICES"] = '0' # only use No.0 GPU
config = tf.ConfigProto()
config.allow_soft_placement = True
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
keras.backend.set_session(sess)
# Get MNIST test data
fashion_mnist = FASHION_MNIST.load_data()
x_train, y_train = fashion_mnist[0]
x_test, y_test = fashion_mnist[1]
# Obtain Image Parameters
img_rows, img_cols, nchannels = x_train.shape[1:4]
nb_classes = y_train.shape[1]
# Define input TF placeholder
x = tf.placeholder(tf.float32, shape=(None, img_rows, img_cols, nchannels))
y = tf.placeholder(tf.float32, shape=(None, nb_classes))
# Define TF model graph
model = cnn_model(img_rows=img_rows,
img_cols=img_cols,
channels=nchannels,
nb_filters=64,
nb_classes=nb_classes)
model = ModelAllConvolutional('model1',
nb_classes,
nb_filters=64,
input_shape=[32, 32, 3])
preds = model(x)
print("Defined TensorFlow model graph.")
def evaluate():
# Evaluate the accuracy of the MNIST model on legitimate test examples
eval_params = {'batch_size': batch_size}
acc = model_eval(sess, x, y, preds, x_test, y_test, args=eval_params)
report.clean_train_clean_eval = acc
# assert X_test.shape[0] == test_end - test_start, X_test.shape
print('Test accuracy on legitimate examples: %0.4f' % acc)
# Train an MNIST model
train_params = {
'nb_epochs': nb_epochs,
'batch_size': batch_size,
'learning_rate': learning_rate,
'train_dir': train_dir,
'filename': filename
}
rng = np.random.RandomState([2017, 8, 30])
if not os.path.exists(train_dir):
os.mkdir(train_dir)
ckpt = tf.train.get_checkpoint_state(train_dir)
print(train_dir, ckpt)
ckpt_path = False if ckpt is None else ckpt.model_checkpoint_path
# wrap = KerasModelWrapper(model)
if load_model and ckpt_path:
saver = tf.train.Saver()
print(ckpt_path)
saver.restore(sess, ckpt_path)
print("Model loaded from: {}".format(ckpt_path))
evaluate()
else:
print("Model was not loaded, training from scratch.")
loss = CrossEntropy(model, smoothing=label_smoothing)
train(sess,
loss,
x_train,
y_train,
evaluate=evaluate,
args=train_params,
rng=rng)
saver = tf.train.Saver(max_to_keep=1)
saver.save(sess,
'{}/cifar10.ckpt'.format(train_dir),
global_step=NB_EPOCHS)
print("model has been saved")
# Calculate training error
if testing:
eval_params = {'batch_size': batch_size}
acc = model_eval(sess, x, y, preds, x_train, y_train, args=eval_params)
report.train_clean_train_clean_eval = acc
# Initialize the Fast Gradient Sign Method (FGSM) attack object and graph
fgsm = FastGradientMethod(model, sess=sess)
fgsm_params = {'eps': 0.2, 'clip_min': 0., 'clip_max': 1.}
print(fgsm_params)
adv_x = fgsm.generate(x, **fgsm_params)
# Consider the attack to be constant
adv_x = tf.stop_gradient(adv_x)
preds_adv = model(adv_x)
# Evaluate the accuracy of the MNIST model on adversarial examples
eval_par = {'batch_size': batch_size}
start_time = time.time()
acc = model_eval(sess, x, y, preds_adv, x_test, y_test, args=eval_par)
print('Test accuracy on adversarial examples: %0.4f' % acc)
end_time = time.time()
print("FGSM attack time is {}\n".format(end_time - start_time))
report.clean_train_adv_eval = acc
# Calculating train error
if testing:
eval_par = {'batch_size': batch_size}
acc = model_eval(sess,
x,
y,
preds_adv,
x_train,
y_train,
args=eval_par)
report.train_clean_train_adv_eval = acc
gc.collect()
return report
all_filename = os.listdir(path)
for filename in all_filename:
pic = io.imread(path + "/" + filename, plugin='matplotlib')
pic = transform.resize(pic, (img_rows, img_cols), preserve_range=True)
my_data.append(pic)
my_data = np.array(my_data)
#要改变的图片格式入口
my_data = my_data.reshape(
(my_data.shape[0], my_data.shape[1], my_data.shape[2], 3))
#训练图片格式入口
print("STEP 3: Start training model...")
x = tf.placeholder(tf.float32, shape=(None, img_rows, img_cols, nchannels))
sess = tf.Session(config=tf.ConfigProto(**config_args))
model = ModelAllConvolutional('model1',
nb_classes,
NB_FILTERS,
input_shape=[32, 32, 3])
preds = model.get_logits(x)
loss = CrossEntropy(model, smoothing=0.1)
train(sess,
loss,
x_train,
y_train,
evaluate=None,
args=train_params,
rng=rng,
var_list=model.get_params())
fgsm = FastGradientMethod(model, sess=sess)
adv_x = fgsm.generate(x, **fgsm_params)
def main(argv):
os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
os.environ["CUDA_VISIBLE_DEVICES"] = "4"
model_file = tf.train.latest_checkpoint(FLAGS.checkpoint_dir)
if model_file is None:
print('No model found')
sys.exit()
source_data_dir = "/home1/machen/dataset/CIFAR-10/cifar-10-batches-py"
cifar = CIFAR(data_dir=source_data_dir,
train_start=0,
train_end=60000,
test_start=0,
test_end=10000)
nb_classes = 10
X_test = cifar.x_test
Y_test = cifar.y_test
assert Y_test.shape[1] == 10.
set_log_level(logging.DEBUG)
with tf.Session() as sess:
x = tf.placeholder(tf.float32, shape=(None, 32, 32, 3))
y = tf.placeholder(tf.float32, shape=(None, 10))
model = ModelAllConvolutional('model1',
nb_classes,
64,
input_shape=[32, 32, 3])
saver = tf.train.Saver()
# Restore the checkpoint
saver.restore(sess, model_file)
nb_samples = FLAGS.nb_samples
attack_params = {
'batch_size': FLAGS.batch_size,
'clip_min': 0.,
'clip_max': 255.
}
if FLAGS.attack_type == 'cwl2':
from cleverhans.attacks import CarliniWagnerL2
attacker = CarliniWagnerL2(model, sess=sess)
attack_params.update({
'binary_search_steps': 1,
'max_iterations': 100,
'learning_rate': 0.1,
'initial_const': 10,
'batch_size': 10
})
else: # eps and eps_iter in range 0-255
attack_params.update({'eps': 8, 'ord': np.inf})
if FLAGS.attack_type == 'fgsm':
from cleverhans.attacks import FastGradientMethod
attacker = FastGradientMethod(model, sess=sess)
elif FLAGS.attack_type == 'pgd':
attack_params.update({'eps_iter': 2, 'nb_iter': 20})
from cleverhans.attacks import MadryEtAl
attacker = MadryEtAl(model, sess=sess)
eval_par = {'batch_size': FLAGS.batch_size}
if FLAGS.sweep:
max_eps = 16
epsilons = np.linspace(1, max_eps, max_eps)
for e in epsilons:
t1 = time.time()
attack_params.update({'eps': e})
x_adv = attacker.generate(x, **attack_params)
preds_adv = model.get_probs(x_adv)
acc = model_eval(sess,
x,
y,
preds_adv,
X_test[:nb_samples],
Y_test[:nb_samples],
args=eval_par)
print('Epsilon %.2f, accuracy on adversarial' % e,
'examples %0.4f\n' % acc)
t2 = time.time()
else:
t1 = time.time()
x_adv = attacker.generate(x, **attack_params)
preds_adv = model.get_probs(x_adv)
acc = model_eval(sess,
x,
y,
preds_adv,
X_test[:nb_samples],
Y_test[:nb_samples],
args=eval_par)
t2 = time.time()
print('Test accuracy on adversarial examples %0.4f\n' % acc)
print("Took", t2 - t1, "seconds")
def main(argv=None):
from cleverhans_tutorials import check_installation
check_installation(__file__)
if not os.path.exists( CONFIG.SAVE_PATH ):
os.makedirs( CONFIG.SAVE_PATH )
save_path_data = CONFIG.SAVE_PATH + 'data/'
if not os.path.exists( save_path_data ):
os.makedirs( save_path_data )
model_path = CONFIG.SAVE_PATH + '../all/' + CONFIG.DATASET + '/'
if not os.path.exists( model_path ):
os.makedirs( model_path )
os.makedirs( model_path + 'data/' )
nb_epochs = FLAGS.nb_epochs
batch_size = FLAGS.batch_size
learning_rate = FLAGS.learning_rate
nb_filters = FLAGS.nb_filters
len_x = int(CONFIG.NUM_TEST/2)
start = time.time()
# Object used to keep track of (and return) key accuracies
report = AccuracyReport()
# Set seeds to improve reproducibility
if CONFIG.DATASET == 'mnist' or CONFIG.DATASET == 'cifar10':
tf.set_random_seed(1234)
np.random.seed(1234)
rd.seed(1234)
elif CONFIG.DATASET == 'moon' or CONFIG.DATASET == 'dims':
tf.set_random_seed(13)
np.random.seed(1234)
rd.seed(0)
# Set logging level to see debug information
set_log_level(logging.DEBUG)
# Create TF session
tf_config = tf.ConfigProto(allow_soft_placement=True,log_device_placement=True)
tf_config.gpu_options.per_process_gpu_memory_fraction = 0.2
sess = tf.Session(config=tf_config)
if CONFIG.DATASET == 'mnist':
# Get MNIST data
mnist = MNIST(train_start=0, train_end=CONFIG.NUM_TRAIN,
test_start=0, test_end=CONFIG.NUM_TEST)
x_train, y_train = mnist.get_set('train')
x_test, y_test = mnist.get_set('test')
elif CONFIG.DATASET == 'cifar10':
# Get CIFAR10 data
data = CIFAR10(train_start=0, train_end=CONFIG.NUM_TRAIN,
test_start=0, test_end=CONFIG.NUM_TEST)
dataset_size = data.x_train.shape[0]
dataset_train = data.to_tensorflow()[0]
dataset_train = dataset_train.map(
lambda x, y: (random_shift(random_horizontal_flip(x)), y), 4)
dataset_train = dataset_train.batch(batch_size)
dataset_train = dataset_train.prefetch(16)
x_train, y_train = data.get_set('train')
x_test, y_test = data.get_set('test')
elif CONFIG.DATASET == 'moon':
# Create a two moon example
X, y = make_moons(n_samples=(CONFIG.NUM_TRAIN+CONFIG.NUM_TEST), noise=0.2,
random_state=0)
X = StandardScaler().fit_transform(X)
x_train1, x_test1, y_train1, y_test1 = train_test_split(X, y,
test_size=(CONFIG.NUM_TEST/(CONFIG.NUM_TRAIN
+CONFIG.NUM_TEST)), random_state=0)
x_train, y_train, x_test, y_test = normalize_reshape_inputs_2d(model_path, x_train1,
y_train1, x_test1,
y_test1)
elif CONFIG.DATASET == 'dims':
X, y = make_moons(n_samples=(CONFIG.NUM_TRAIN+CONFIG.NUM_TEST), noise=0.2,
random_state=0)
X = StandardScaler().fit_transform(X)
x_train1, x_test1, y_train1, y_test1 = train_test_split(X, y,
test_size=(CONFIG.NUM_TEST/(CONFIG.NUM_TRAIN
+CONFIG.NUM_TEST)), random_state=0)
x_train2, y_train, x_test2, y_test = normalize_reshape_inputs_2d(model_path, x_train1,
y_train1,x_test1,
y_test1)
x_train, x_test = add_noise_and_QR(x_train2, x_test2, CONFIG.NUM_DIMS)
np.save(os.path.join(save_path_data, 'x_test'), x_test)
np.save(os.path.join(save_path_data, 'y_test'), y_test)
# Use Image Parameters
img_rows, img_cols, nchannels = x_train.shape[1:4]
nb_classes = y_train.shape[1]
# Define input TF placeholder
x = tf.placeholder(tf.float32, shape=(None, img_rows, img_cols,
nchannels))
y = tf.placeholder(tf.float32, shape=(None, nb_classes))
# Train an model
train_params = {
'nb_epochs': nb_epochs,
'batch_size': batch_size,
'learning_rate': learning_rate
}
eval_params = {'batch_size': 1}
rng = np.random.RandomState([2017, 8, 30])
with open(CONFIG.SAVE_PATH + 'acc_param.txt', 'a') as fi:
def do_eval(adv_x, preds, x_set, y_set, report_key):
acc, pred_np, adv_x_np = model_eval(sess, x, y, preds, adv_x, nb_classes, x_set,
y_set, args=eval_params)
setattr(report, report_key, acc)
if report_key:
print('Accuracy on %s examples: %0.4f' % (report_key, acc), file=fi)
return pred_np, adv_x_np
if CONFIG.DATASET == 'mnist':
trained_model_path = model_path + 'data/trained_model'
model = ModelBasicCNN('model1', nb_classes, nb_filters)
elif CONFIG.DATASET == 'cifar10':
trained_model_path = model_path + 'data/trained_model'
model = ModelAllConvolutional('model1', nb_classes, nb_filters,
input_shape=[32, 32, 3])
elif CONFIG.DATASET == 'moon':
trained_model_path = model_path + 'data/trained_model'
model = ModelMLP('model1', nb_classes)
elif CONFIG.DATASET == 'dims':
trained_model_path = save_path_data + 'trained_model'
model = ModelMLP_dyn('model1', nb_classes, CONFIG.NUM_DIMS)
preds = model.get_logits(x)
loss = CrossEntropy(model, smoothing=0.1)
def evaluate():
_, _ = do_eval(x, preds, x_test, y_test, 'test during train')
if os.path.isfile( trained_model_path + '.index' ):
tf_model_load(sess, trained_model_path)
else:
if CONFIG.DATASET == 'mnist':
train(sess, loss, x_train, y_train, evaluate=evaluate,
args=train_params, rng=rng, var_list=model.get_params())
elif CONFIG.DATASET == 'cifar10':
train(sess, loss, None, None,
dataset_train=dataset_train, dataset_size=dataset_size,
evaluate=evaluate, args=train_params, rng=rng,
var_list=model.get_params())
elif CONFIG.DATASET == 'moon':
train_2d(sess, loss, x, y, x_train, y_train, save=False, evaluate=evaluate,
args=train_params, rng=rng, var_list=model.get_params())
elif CONFIG.DATASET == 'dims':
train_2d(sess, loss, x, y, x_train, y_train, evaluate=evaluate,
args=train_params, rng=rng, var_list=model.get_params())
saver = tf.train.Saver()
saver.save(sess, trained_model_path)
# Evaluate the accuracy on test examples
if os.path.isfile( save_path_data + 'logits_zero_attacked.npy' ):
logits_0 = np.load(save_path_data + 'logits_zero_attacked.npy')
else:
_, _ = do_eval(x, preds, x_train, y_train, 'train')
logits_0, _ = do_eval(x, preds, x_test, y_test, 'test')
np.save(os.path.join(save_path_data, 'logits_zero_attacked'), logits_0)
if CONFIG.DATASET == 'moon':
num_grid_points = 5000
if os.path.isfile( model_path + 'data/images_mesh' + str(num_grid_points) + '.npy' ):
x_mesh = np.load(model_path + 'data/images_mesh' + str(num_grid_points) + '.npy')
logits_mesh = np.load(model_path + 'data/logits_mesh' + str(num_grid_points) + '.npy')
else:
xx, yy = np.meshgrid(np.linspace(0, 1, num_grid_points), np.linspace(0, 1, num_grid_points))
x_mesh1 = np.stack([np.ravel(xx), np.ravel(yy)]).T
y_mesh1 = np.ones((x_mesh1.shape[0]),dtype='int64')
x_mesh, y_mesh, _, _ = normalize_reshape_inputs_2d(model_path, x_mesh1, y_mesh1)
logits_mesh, _ = do_eval(x, preds, x_mesh, y_mesh, 'mesh')
x_mesh = np.squeeze(x_mesh)
np.save(os.path.join(model_path, 'data/images_mesh'+str(num_grid_points)), x_mesh)
np.save(os.path.join(model_path, 'data/logits_mesh'+str(num_grid_points)), logits_mesh)
points_x = x_test[:len_x]
points_y = y_test[:len_x]
points_x_bar = x_test[len_x:]
points_y_bar = y_test[len_x:]
# Initialize the CW attack object and graph
cw = CarliniWagnerL2(model, sess=sess)
# first attack
attack_params = {
'learning_rate': CONFIG.CW_LEARNING_RATE,
'max_iterations': CONFIG.CW_MAX_ITERATIONS
}
if CONFIG.DATASET == 'moon':
out_a = compute_polytopes_a(x_mesh, logits_mesh, model_path)
attack_params['const_a_min'] = out_a
attack_params['const_a_max'] = 100
adv_x = cw.generate(x, **attack_params)
if os.path.isfile( save_path_data + 'images_once_attacked.npy' ):
adv_img_1 = np.load(save_path_data + 'images_once_attacked.npy')
logits_1 = np.load(save_path_data + 'logits_once_attacked.npy')
else:
#Evaluate the accuracy on adversarial examples
preds_adv = model.get_logits(adv_x)
logits_1, adv_img_1 = do_eval(adv_x, preds_adv, points_x_bar, points_y_bar,
'test once attacked')
np.save(os.path.join(save_path_data, 'images_once_attacked'), adv_img_1)
np.save(os.path.join(save_path_data, 'logits_once_attacked'), logits_1)
# counter attack
attack_params['max_iterations'] = 1024
if CONFIG.DATASET == 'moon':
out_alpha2 = compute_epsilons_balls_alpha(x_mesh, np.squeeze(x_test),
np.squeeze(adv_img_1), model_path,
CONFIG.SAVE_PATH)
attack_params['learning_rate'] = out_alpha2
attack_params['const_a_min'] = -1
attack_params['max_iterations'] = 2048
plot_data(np.squeeze(adv_img_1), logits_1, CONFIG.SAVE_PATH+'data_pred1.png', x_mesh,
logits_mesh)
adv_adv_x = cw.generate(x, **attack_params)
x_k = np.concatenate((points_x, adv_img_1), axis=0)
y_k = np.concatenate((points_y, logits_1), axis=0)
if os.path.isfile( save_path_data + 'images_twice_attacked.npy' ):
adv_img_2 = np.load(save_path_data + 'images_twice_attacked.npy')
logits_2 = np.load(save_path_data + 'logits_twice_attacked.npy')
else:
# Evaluate the accuracy on adversarial examples
preds_adv_adv = model.get_logits(adv_adv_x)
logits_2, adv_img_2 = do_eval(adv_adv_x, preds_adv_adv, x_k, y_k,
'test twice attacked')
np.save(os.path.join(save_path_data, 'images_twice_attacked'), adv_img_2)
np.save(os.path.join(save_path_data, 'logits_twice_attacked'), logits_2)
if CONFIG.DATASET == 'moon':
plot_data(np.squeeze(adv_img_2[:len_x]), logits_2[:len_x],
CONFIG.SAVE_PATH+'data_pred2.png', x_mesh, logits_mesh)
plot_data(np.squeeze(adv_img_2[len_x:]), logits_2[len_x:],
CONFIG.SAVE_PATH+'data_pred12.png', x_mesh, logits_mesh)
test_balls(np.squeeze(x_k), np.squeeze(adv_img_2), logits_0, logits_1, logits_2,
CONFIG.SAVE_PATH)
compute_returnees(logits_0[len_x:], logits_1, logits_2[len_x:], logits_0[:len_x],
logits_2[:len_x], CONFIG.SAVE_PATH)
if x_test.shape[-1] > 1:
num_axis=(1,2,3)
else:
num_axis=(1,2)
D_p = np.squeeze(np.sqrt(np.sum(np.square(points_x-adv_img_2[:len_x]), axis=num_axis)))
D_p_p = np.squeeze(np.sqrt(np.sum(np.square(adv_img_1-adv_img_2[len_x:]),
axis=num_axis)))
D_p_mod, D_p_p_mod = modify_D(D_p, D_p_p, logits_0[len_x:], logits_1, logits_2[len_x:],
logits_0[:len_x], logits_2[:len_x])
if D_p_mod != [] and D_p_p_mod != []:
plot_violins(D_p_mod, D_p_p_mod, CONFIG.SAVE_PATH)
threshold_evaluation(D_p_mod, D_p_p_mod, CONFIG.SAVE_PATH)
_ = compute_auroc(D_p_mod, D_p_p_mod, CONFIG.SAVE_PATH)
plot_results_models(len_x, CONFIG.DATASET, CONFIG.SAVE_PATH)
print('Time needed:', time.time()-start)
return report
def mnist_tutorial_cw(train_start=0, train_end=60000, test_start=0,
test_end=10000, viz_enabled=VIZ_ENABLED,
nb_epochs=NB_EPOCHS, batch_size=BATCH_SIZE,
source_samples=SOURCE_SAMPLES,
learning_rate=LEARNING_RATE,
attack_iterations=ATTACK_ITERATIONS,
model_path=MODEL_PATH,
targeted=TARGETED):
"""
MNIST tutorial for Carlini and Wagner's attack
:param train_start: index of first training set example
:param train_end: index of last training set example
:param test_start: index of first test set example
:param test_end: index of last test set example
:param viz_enabled: (boolean) activate plots of adversarial examples
:param nb_epochs: number of epochs to train model
:param batch_size: size of training batches
:param nb_classes: number of output classes
:param source_samples: number of test inputs to attack
:param learning_rate: learning rate for training
:param model_path: path to the model file
:param targeted: should we run a targeted attack? or untargeted?
:return: an AccuracyReport object
"""
# Object used to keep track of (and return) key accuracies
report = AccuracyReport()
# Set TF random seed to improve reproducibility
tf.set_random_seed(1234)
# Create TF session
config_args = dict(intra_op_parallelism_threads=1)
config_args["gpu_options"] = tf.GPUOptions(allow_growth=True)
sess = tf.Session(config=tf.ConfigProto(**config_args))
print("Created TensorFlow session.")
set_log_level(logging.DEBUG)
# Get MNIST test data
mnist = MNIST(DATA_DIR, train_start=train_start, train_end=train_end,
test_start=test_start, test_end=test_end)
x_train, y_train = mnist.get_set('train')
x_test, y_test = mnist.get_set('test')
# Obtain Image Parameters
img_rows, img_cols, nchannels = x_train.shape[1:4]
nb_classes = y_train.shape[1]
# Define input TF placeholder
x = tf.placeholder(tf.float32, shape=(None, img_rows, img_cols,
nchannels))
y = tf.placeholder(tf.float32, shape=(None, nb_classes))
nb_filters = 64
# Define TF model graph
model = ModelAllConvolutional('model1', nb_classes, nb_filters,
input_shape=[28, 28, 1])
preds = model.get_logits(x)
loss = CrossEntropy(model, smoothing=0.1)
print("Defined TensorFlow model graph.")
###########################################################################
# Training the model using TensorFlow
###########################################################################
# Train an MNIST model
train_params = {
'nb_epochs': nb_epochs,
'batch_size': batch_size,
'learning_rate': learning_rate,
'filename': os.path.split(model_path)[-1]
}
rng = np.random.RandomState([2017, 8, 30])
# check if we've trained before, and if we have, use that pre-trained model
if os.path.exists(model_path + ".meta"):
tf_model_load(sess, model_path)
else:
train(sess, loss, x_train, y_train, args=train_params, rng=rng)
saver = tf.train.Saver()
saver.save(sess, model_path)
# Evaluate the accuracy of the MNIST model on legitimate test examples
eval_params = {'batch_size': batch_size}
accuracy = model_eval(sess, x, y, preds, x_test, y_test, args=eval_params)
assert x_test.shape[0] == test_end - test_start, x_test.shape
print('Test accuracy on legitimate test examples: {0}'.format(accuracy))
report.clean_train_clean_eval = accuracy
###########################################################################
# Craft adversarial examples using Carlini and Wagner's approach
###########################################################################
nb_adv_per_sample = str(nb_classes - 1) if targeted else '1'
print('Crafting ' + str(source_samples) + ' * ' + nb_adv_per_sample +
' adversarial examples')
print("This could take some time ...")
# Instantiate a CW attack object
cw = CarliniWagnerL2(model, sess=sess)
if viz_enabled:
assert source_samples == nb_classes
idxs = [np.where(np.argmax(y_test, axis=1) == i)[0][0]
for i in range(nb_classes)]
if targeted:
if viz_enabled:
# Initialize our array for grid visualization
grid_shape = (nb_classes, nb_classes, img_rows, img_cols,
nchannels)
grid_viz_data = np.zeros(grid_shape, dtype='f')
adv_inputs = np.array(
[[instance] * nb_classes for instance in x_test[idxs]],
dtype=np.float32)
else:
adv_inputs = np.array(
[[instance] * nb_classes for
instance in x_test[:source_samples]], dtype=np.float32)
one_hot = np.zeros((nb_classes, nb_classes))
one_hot[np.arange(nb_classes), np.arange(nb_classes)] = 1
adv_inputs = adv_inputs.reshape(
(source_samples * nb_classes, img_rows, img_cols, nchannels))
adv_ys = np.array([one_hot] * source_samples,
dtype=np.float32).reshape((source_samples *
nb_classes, nb_classes))
yname = "y_target"
else:
if viz_enabled:
# Initialize our array for grid visualization
grid_shape = (nb_classes, 2, img_rows, img_cols, nchannels)
grid_viz_data = np.zeros(grid_shape, dtype='f')
adv_inputs = x_test[idxs]
else:
adv_inputs = x_test[:source_samples]
adv_ys = None
yname = "y"
if targeted:
cw_params_batch_size = source_samples * nb_classes
else:
cw_params_batch_size = source_samples
cw_params = {'binary_search_steps': 1,
yname: adv_ys,
'max_iterations': attack_iterations,
'learning_rate': CW_LEARNING_RATE,
'batch_size': cw_params_batch_size,
'initial_const': 10}
adv = cw.generate_np(adv_inputs,
**cw_params)
eval_params = {'batch_size': np.minimum(nb_classes, source_samples)}
if targeted:
adv_accuracy = model_eval(
sess, x, y, preds, adv, adv_ys, args=eval_params)
else:
if viz_enabled:
err = model_eval(sess, x, y, preds, adv, y_test[idxs], args=eval_params)
adv_accuracy = 1 - err
else:
err = model_eval(sess, x, y, preds, adv, y_test[:source_samples],
args=eval_params)
adv_accuracy = 1 - err
if viz_enabled:
for j in range(nb_classes):
if targeted:
for i in range(nb_classes):
grid_viz_data[i, j] = adv[i * nb_classes + j]
else:
grid_viz_data[j, 0] = adv_inputs[j]
grid_viz_data[j, 1] = adv[j]
print(grid_viz_data.shape)
print('--------------------------------------')
# Compute the number of adversarial examples that were successfully found
print('Avg. rate of successful adv. examples {0:.4f}'.format(adv_accuracy))
report.clean_train_adv_eval = 1. - adv_accuracy
# Compute the average distortion introduced by the algorithm
percent_perturbed = np.mean(np.sum((adv - adv_inputs) ** 2,
axis=(1, 2, 3)) ** .5)
print('Avg. L_2 norm of perturbations {0:.4f}'.format(percent_perturbed))
# Close TF session
sess.close()
# Finally, block & display a grid of all the adversarial examples
if viz_enabled:
_ = grid_visual(grid_viz_data)
return report
def build_model_vaegan(self):
self.z1_mean, self.z1_sigm = self.Encode1(self.x_input)
self.z1_x = tf.add( self.z1_mean, tf.sqrt(tf.exp(self.z1_sigm))*self.ep1)
self.x_input_sobel = tf.image.sobel_edges(self.x_input)
self.x_input_sobel = tf.reshape(self.x_input_sobel, [64,128,128,6])
self.x_out = self.generate1(self.x_input_sobel, self.z1_x, reuse=False)
self.x_filt2 = self.generate1(self.x_input_sobel, self.z1_mean, reuse=True)
self.model_classifier_logits = ModelAllConvolutional('model1', 11, 64, input_shape=[self.output_size,self.output_size,self.channel])
self.model_classifier_percept = ModelAllConvolutional1('model2', 11, 64, input_shape=[self.output_size,self.output_size,self.channel])
#tanh o/p -1 to 1
self.logits_x_true = self.model_classifier_logits.get_logits((self.x_true+1)*0.5)
self.percept_x_true = self.model_classifier_percept.get_logits((self.x_true+1)*0.5)
#self.pred_x_true = tf.nn.softmax(self.logits_x_true)
self.pred_x_true = self.model_classifier_percept.get_probs((self.x_true+1)*0.5)
self.logits_x_out = self.model_classifier_logits.get_logits((self.x_out+1)*0.5)
self.percept_x_out = self.model_classifier_percept.get_logits((self.x_out+1)*0.5)
self.pred_x_out = tf.nn.softmax(self.logits_x_out)
self.logits_x_filt2 = self.model_classifier_logits.get_logits((self.x_filt2+1)*0.5)
self.pred_x_filt2 = tf.nn.softmax(self.logits_x_filt2)
self.cl_loss_x_true = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits = self.logits_x_true, labels = self.labels))
self.cl_loss_x_out = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits = self.logits_x_out , labels = self.labels))
self.cl_loss_x_true = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits = self.logits_x_true, labels = self.labels))
self.kl1_loss = self.KL_loss(self.z1_mean, self.z1_sigm)/(self.latent_dim*self.batch_size)
self.Loss_vae1_pixel = tf.reduce_mean(tf.square(tf.subtract(self.x_out, self.x_true))) + tf.reduce_mean(tf.abs(tf.subtract(self.x_out, self.x_true)))
self.Loss_vae1_percept = tf.reduce_mean(tf.square(tf.subtract(self.percept_x_out, self.percept_x_true)))
self.Loss_vae1_logits = tf.reduce_mean(tf.square(tf.subtract(self.logits_x_out, self.logits_x_true)))
#For encode
self.encode1_loss = 1*self.kl1_loss + 10*self.Loss_vae1_pixel + 0*self.cl_loss_x_out + 0*self.Loss_vae1_logits + 1000*self.Loss_vae1_percept
#for Gen
self.G1_loss = 10*self.Loss_vae1_pixel + 0*self.cl_loss_x_out + 0*self.Loss_vae1_logits + 1000*self.Loss_vae1_percept
t_vars = tf.trainable_variables()
self.log_vars.append(("encode1_loss", self.encode1_loss))
self.log_vars.append(("generator1_loss", self.G1_loss))
self.g1_vars = [var for var in t_vars if 'VAE_gen1' in var.name]
self.e1_vars = [var for var in t_vars if 'VAE_e1_' in var.name]
self.saver = tf.train.Saver()
for k, v in self.log_vars:
tf.summary.scalar(k, v)
print('Model is Built')
class vaegan(object):
#build model
def __init__(self, batch_size, max_iters, repeat, model_path, latent_dim, sample_path, log_dir, learnrate_init):
self.batch_size = batch_size
self.max_iters = max_iters
self.repeat_num = repeat
self.saved_model_path = model_path
self.latent_dim = latent_dim
self.sample_path = sample_path
self.log_dir = log_dir
self.learn_rate_init = learnrate_init
self.log_vars = []
self.channel = 3
self.output_size = 128
self.x_input = tf.placeholder(tf.float32, [self.batch_size, self.output_size, self.output_size, 3])
self.x_true = tf.placeholder(tf.float32, [self.batch_size, self.output_size, self.output_size, self.channel])
self.labels = tf.placeholder(tf.float32, [self.batch_size, 11])
self.ep1 = tf.random_normal(shape=[self.batch_size, self.latent_dim])
self.zp1 = tf.random_normal(shape=[self.batch_size, self.latent_dim])
self.ep2 = tf.random_normal(shape=[self.batch_size, self.latent_dim])
self.zp2 = tf.random_normal(shape=[self.batch_size, self.latent_dim])
self.keep_prob = tf.placeholder_with_default(1.0, shape=())
print('Data Loading Begins')
y_train=[]
x_train1=[]
for dirs in os.listdir('/home/manu_kohli/wbc/cam3/trainset/'):
for files in os.listdir('/home/manu_kohli/wbc/cam3/trainset/'+dirs):
y_train.append(int(dirs))
x_train1.append(np.array(PIL.Image.open('/home/manu_kohli/wbc/cam3/trainset/'+dirs+'/'+files)))
#x_train1 =np.asarray(x_train1)/255.0
cam3_train_data=[]
cam3_train_label=[]
l=list(range(0,len(y_train)))
l=np.asarray(l)
np.random.shuffle(l)
for i in l:
cam3_train_data.append(x_train1[i])
cam3_train_label.append(y_train[i])
x_train1=cam3_train_data
y_train=cam3_train_label
x_train1 = np.asarray(x_train1)/127.5
x_train1 =x_train1 - 1.
y_train = np.asarray(y_train)
#y_train = toOneHot(y_train)
y_train= to_categorical(y_train, num_classes=11)
# x_train1 = np.load( '/home/vinay/projects/Sigtuple/CreateData/DataAugmentation/X_Train.npy').astype('float32')
# y_train = np.load( '/home/vinay/projects/Sigtuple/CreateData/DataAugmentation/Y_Train.npy')
# x_train1_1 = np.load('/home/vinay/projects/Sigtuple/CreateData/DataAugmentation/X_Test.npy').astype('float32')
# y_train_1 = np.load('/home/vinay/projects/Sigtuple/CreateData/DataAugmentation/Y_Test.npy')
# x_train1_2 = np.load( '/home/vinay/projects/Sigtuple/CameraInvariance/Cam3Classifier/Data_Augmentation/X_Train_extra.npy').astype('float32')
# y_train_2 = np.load( '/home/vinay/projects/Sigtuple/CameraInvariance/Cam3Classifier/Data_Augmentation/Y_Train_extra.npy')
# x_train1 = np.append(x_train1, x_train1_2,axis =0)
# y_train = np.append(y_train, y_train_2,axis =0)
# x_train1 = np.concatenate((x_train1, x_train1_1), axis=0)
# y_train = np.concatenate((y_train, y_train_1), axis=0)
x_test1_cam3 = []
y_test_cam3 = []
for dirs in os.listdir('/home/manu_kohli/wbc/cam3/testset/'):
for files in os.listdir('/home/manu_kohli/wbc/cam3/testset/'+dirs):
y_test_cam3.append(int(dirs))
x_test1_cam3.append(np.array(PIL.Image.open('/home/manu_kohli/wbc/cam3/testset/'+dirs+'/'+files)))
cam3_test_data=[]
cam3_test_label=[]
l=list(range(0,len(y_test_cam3)))
l=np.asarray(l)
np.random.shuffle(l)
for i in l:
cam3_test_data.append(x_test1_cam3[i])
cam3_test_label.append(y_test_cam3[i])
x_test1_cam3 = cam3_test_data
y_test_cam3 = cam3_test_label
y_test_cam3= to_categorical(y_test_cam3, num_classes=11)
#y_test_cam3 = toOneHot(np.asarray(y_test_cam3))
#x_test1_cam3=np.asarray(x_test1_cam3)/255.0
x_test1_cam3 = np.asarray(x_test1_cam3)/127.5
x_test1_cam3 =x_test1_cam3 - 1.
x_test1=[]
y_test =[]
for dirs in os.listdir('/home/manu_kohli/wbc/cam2/combine_train_test_cam2/'):
for files in os.listdir('/home/manu_kohli/wbc/cam2/combine_train_test_cam2/'+dirs):
y_test.append(int(dirs))
x_test1.append(np.array(PIL.Image.open('/home/manu_kohli/wbc/cam2/combine_train_test_cam2/'+dirs+'/'+files)))
# x_test1 = np.load('/home/vinay/projects/Sigtuple/CreateData/cam2_images.npy').astype('float32')/255
# y_test = np.load('/home/vinay/projects/Sigtuple/CreateData/cam2_labels.npy')
cam2_data=[]
cam2_label=[]
l=list(range(0,len(y_test)))
l=np.asarray(l)
np.random.shuffle(l)
for i in l:
cam2_data.append(x_test1[i])
cam2_label.append(y_test[i])
x_test1 = cam2_data
y_test = cam2_label
y_test= to_categorical(y_test, num_classes=11)
#y_test = toOneHot(np.asarray(y_test))
# x_test1=np.asarray(x_test1)/255.0
x_test1 = np.asarray(x_test1)/127.5
x_test1 =x_test1 - 1.
# x_test1_cam3 = np.load('/home/vinay/projects/Sigtuple/CreateData/cam3_images.npy').astype('float32')/255
# y_test_cam3 = np.load('/home/vinay/projects/Sigtuple/CreateData/cam3_labels.npy')
# y_test_cam3 = toOneHot(y_test_cam3)
#print(x_train1.shape, y_train.shape)
#print(x_test1.shape, y_test.shape)
#x_train = np.zeros([x_train1.shape[0], self.output_size,self.output_size,self.channel])
#x_test = np.zeros([x_test1.shape[0], self.output_size,self.output_size,self.channel])
#x_test_cam3 = np.zeros([x_test1_cam3.shape[0], self.output_size,self.output_size,self.channel])
# x_train[:,:,:,0] = x_train1[:,:,:,2]
# x_train[:,:,:,1] = x_train1[:,:,:,1]
# x_train[:,:,:,2] = x_train1[:,:,:,0]
# x_test[:,:,:,0] = x_test1[:,:,:,2]
# x_test[:,:,:,1] = x_test1[:,:,:,1]
# x_test[:,:,:,2] = x_test1[:,:,:,0]
# x_test_cam3[:,:,:,0] = x_test1_cam3[:,:,:,2]
# x_test_cam3[:,:,:,1] = x_test1_cam3[:,:,:,1]
# x_test_cam3[:,:,:,2] = x_test1_cam3[:,:,:,0]
x_train = np.float32(x_train1).reshape([-1,self.output_size,self.output_size,self.channel])
x_test = np.float32(x_test1).reshape([-1,self.output_size,self.output_size,self.channel])
x_test_cam3 = np.float32(x_test1_cam3).reshape([-1,self.output_size,self.output_size,self.channel])
print(x_train.shape, y_train.shape)
print(x_test.shape, y_test.shape)
print(x_test_cam3.shape, y_test_cam3.shape)
print(np.amin(x_train), np.amin( x_test ), np.amin(x_test_cam3))
print(np.amax(x_train), np.amax( x_test ), np.amax(x_test_cam3))
TrainDataSize = x_train.shape[0]
TestDataSize = x_test.shape[0]
self.TrainDataSize = TrainDataSize
self.TestDataSize = TestDataSize
self.TestDataSize_cam3 = x_test_cam3.shape[0]
self.X_Real_Test = x_test
self.X_Real_Train = x_train
self.X_Real_Test_cam3 = x_test_cam3
self.Y_train = y_train
self.Y_test = y_test
self.Y_test_cam3 = y_test_cam3
# self.X_Real_Train = self.X_Real_Train*2 - 1
# self.X_Real_Test = self.X_Real_Test*2 - 1
# self.X_Real_Test_cam3 = self.X_Real_Test_cam3*2 - 1
print('Max', np.max(self.X_Real_Train))
print('Min', np.min(self.X_Real_Train))
print('Data Loading Completed')
def build_model_vaegan(self):
self.z1_mean, self.z1_sigm = self.Encode1(self.x_input)
self.z1_x = tf.add( self.z1_mean, tf.sqrt(tf.exp(self.z1_sigm))*self.ep1)
self.x_input_sobel = tf.image.sobel_edges(self.x_input)
self.x_input_sobel = tf.reshape(self.x_input_sobel, [64,128,128,6])
self.x_out = self.generate1(self.x_input_sobel, self.z1_x, reuse=False)
self.x_filt2 = self.generate1(self.x_input_sobel, self.z1_mean, reuse=True)
self.model_classifier_logits = ModelAllConvolutional('model1', 11, 64, input_shape=[self.output_size,self.output_size,self.channel])
self.model_classifier_percept = ModelAllConvolutional1('model2', 11, 64, input_shape=[self.output_size,self.output_size,self.channel])
#tanh o/p -1 to 1
self.logits_x_true = self.model_classifier_logits.get_logits((self.x_true+1)*0.5)
self.percept_x_true = self.model_classifier_percept.get_logits((self.x_true+1)*0.5)
#self.pred_x_true = tf.nn.softmax(self.logits_x_true)
self.pred_x_true = self.model_classifier_percept.get_probs((self.x_true+1)*0.5)
self.logits_x_out = self.model_classifier_logits.get_logits((self.x_out+1)*0.5)
self.percept_x_out = self.model_classifier_percept.get_logits((self.x_out+1)*0.5)
self.pred_x_out = tf.nn.softmax(self.logits_x_out)
self.logits_x_filt2 = self.model_classifier_logits.get_logits((self.x_filt2+1)*0.5)
self.pred_x_filt2 = tf.nn.softmax(self.logits_x_filt2)
self.cl_loss_x_true = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits = self.logits_x_true, labels = self.labels))
self.cl_loss_x_out = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits = self.logits_x_out , labels = self.labels))
self.cl_loss_x_true = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits = self.logits_x_true, labels = self.labels))
self.kl1_loss = self.KL_loss(self.z1_mean, self.z1_sigm)/(self.latent_dim*self.batch_size)
self.Loss_vae1_pixel = tf.reduce_mean(tf.square(tf.subtract(self.x_out, self.x_true))) + tf.reduce_mean(tf.abs(tf.subtract(self.x_out, self.x_true)))
self.Loss_vae1_percept = tf.reduce_mean(tf.square(tf.subtract(self.percept_x_out, self.percept_x_true)))
self.Loss_vae1_logits = tf.reduce_mean(tf.square(tf.subtract(self.logits_x_out, self.logits_x_true)))
#For encode
self.encode1_loss = 1*self.kl1_loss + 10*self.Loss_vae1_pixel + 0*self.cl_loss_x_out + 0*self.Loss_vae1_logits + 1000*self.Loss_vae1_percept
#for Gen
self.G1_loss = 10*self.Loss_vae1_pixel + 0*self.cl_loss_x_out + 0*self.Loss_vae1_logits + 1000*self.Loss_vae1_percept
t_vars = tf.trainable_variables()
self.log_vars.append(("encode1_loss", self.encode1_loss))
self.log_vars.append(("generator1_loss", self.G1_loss))
self.g1_vars = [var for var in t_vars if 'VAE_gen1' in var.name]
self.e1_vars = [var for var in t_vars if 'VAE_e1_' in var.name]
self.saver = tf.train.Saver()
for k, v in self.log_vars:
tf.summary.scalar(k, v)
print('Model is Built')
#do train
def train(self):
global_step = tf.Variable(0, trainable=False)
add_global = global_step.assign_add(1)
new_learning_rate = tf.train.exponential_decay(self.learn_rate_init, global_step=global_step, decay_steps=10000,
decay_rate=0.98)
#for G1
trainer_G1 = tf.train.RMSPropOptimizer(learning_rate=new_learning_rate)
#trainer_G1 = tf.train.RMSPropOptimizer(learning_rate=self.learn_rate_init)
#trainer_G1 = tf.train.AdamOptimizer(learning_rate=new_learning_rate)
gradients_G1 = trainer_G1.compute_gradients(self.G1_loss, var_list=self.g1_vars)
opti_G1 = trainer_G1.apply_gradients(gradients_G1)
#for E1
trainer_E1 = tf.train.RMSPropOptimizer(learning_rate=new_learning_rate)
#trainer_E1 = tf.train.RMSPropOptimizer(learning_rate=self.learn_rate_init)
#trainer_E1 = tf.train.AdamOptimizer(learning_rate=new_learning_rate)
gradients_E1 = trainer_E1.compute_gradients(self.encode1_loss, var_list=self.e1_vars)
opti_E1 = trainer_E1.apply_gradients(gradients_E1)
init = tf.global_variables_initializer()
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
with tf.Session(config=config) as sess:
#changed restoring of weights.
ckpt = tf.train.get_checkpoint_state('/home/manu_kohli/vae_classifier_weights/Classifier/checkpoint')
ckpt_path = ckpt.model_checkpoint_path
sess.run(init)
self.saver.restore(sess , self.saved_model_path)
#print(tf.trainable_variables(),'tf.trainable_variables()')
#saver = tf.train.Saver([var for var in tf.trainable_variables() if var.name.startswith('model1')])
#print(ckpt_path)
#saver.restore(sess, ckpt_path)
##self.saver.save(sess , self.saved_model_path)
print('Creating a Replica of s1 onto s2')
s1_vars1 = [var.name for var in tf.trainable_variables() if 'model1' in var.name]
s2_vars1 = [var for var in tf.trainable_variables() if 'model2' in var.name]
dictionary = {}
for i in range(len(s2_vars1)):
dictionary[s1_vars1[i][0:-2]] = s2_vars1[i]
saver_new = tf.train.Saver(var_list=dictionary)
#saver_new.restore(sess, ckpt_path)
##self.saver.save(sess , ckpt.model_checkpoint_path)
print('******************')
print(' ')
print(' ')
print('Plain VAE Training Begins')
print(' ')
print(' ')
print('******************')
step = 0
g_acc=87.0
batchNum = 0
step=0
while step <= 100000:
next_x_images = self.X_Real_Train[batchNum*self.batch_size:(batchNum+1)*self.batch_size]
next_y_images = self.Y_train[batchNum*self.batch_size:(batchNum+1)*self.batch_size]
batchNum = batchNum +1
#print(batchNum*self.batch_size)
if(((batchNum+1)%170)==0):
idx = np.random.permutation(len(self.X_Real_Train))
self.X_Real_Train,self.Y_train = self.X_Real_Train[idx], self.Y_train[idx]
batchNum = 0
print('data exhausted')
#print(idx)
#print(self.X_Real_Train.shape, self.Y_train.shape)
#print(batchNum)
#print(next_y_images)
fd ={self.keep_prob:1, self.x_input: next_x_images, self.x_true: next_x_images, self.labels: next_y_images}
sess.run(opti_E1, feed_dict=fd)
sess.run(opti_G1, feed_dict=fd)
new_learn_rate = sess.run(new_learning_rate)
if new_learn_rate > 0.00005:
sess.run(add_global)
if np.mod(step , 100) == 0 and step != 0:
# for iter in range(200):
# print('step', step)
#print('model saved: ', self.saved_model_path)
#self.saver.save(sess , self.saved_model_path, global_step=step)
print('lr:', new_learn_rate)
k1, e1, l11, l12, l13, cl, g1 = sess.run([self.kl1_loss , self.encode1_loss,self.Loss_vae1_pixel,self.Loss_vae1_percept, self.Loss_vae1_logits,self.cl_loss_x_out,self.G1_loss],feed_dict=fd)
print('E1_loss_KL_Loss: ',k1)
print('E1_loss_Total: ', e1)
print('G1_loss_MSE: ', l11, 10*l11)
print('G1_loss_Percept: ', l12, 0*l12)
print('G1_loss_Logits: ', l13, 0*l13)
print('G1_loss_CL: ', cl, 1*cl)
print('G1_loss_Total: ', g1)
Preddiction = np.zeros([self.TestDataSize_cam3,11])
for i in range(np.int(self.TestDataSize_cam3/self.batch_size)):
next_x_images = self.X_Real_Test_cam3[i*self.batch_size:(i+1)*self.batch_size]
pred = sess.run(self.pred_x_filt2, feed_dict={self.x_input: next_x_images, self.keep_prob:1})
Preddiction[i*self.batch_size:(i+1)*self.batch_size] = pred.reshape([64,11])
x_filt = sess.run(self.x_filt2, feed_dict={self.x_input: next_x_images, self.keep_prob:1})
x_filt_percept = sess.run(self.percept_x_out, feed_dict={self.x_input: next_x_images, self.keep_prob:1})
print('shape:', x_filt_percept.shape)
if (step == 100):
np.save('Data/x_cam3_test.npy',next_x_images)
name = 'Data/x_filt__' + str(step) + '_.npy'
np.save(name,x_filt)
# print('Full Filtered Real Train Example Acc = ',getAcc(Preddiction[0:150*64], self.Y_test_cam3[0:150*64]))
# print('Full Filtered Real Test Example Acc = ',getAcc(Preddiction[150*64:], self.Y_test_cam3[150*64:]))
accs,l_acc = getAcc(Preddiction, self.Y_test_cam3)
print('Full Filtered Real Test Example Acc = ',accs,l_acc)
if(l_acc>g_acc):
print('model saved: ', '/home/manu_kohli/vae_classifier_weights/VAE/itr_model_2/model.cpkt')
self.saver.save(sess , '/home/manu_kohli/vae_classifier_weights/VAE/itr_model_2/model.cpkt', global_step=step)
g_acc= l_acc
Preddiction = np.zeros([self.TrainDataSize,11])
for i in range(np.int(self.TrainDataSize/self.batch_size)):
next_x_images = self.X_Real_Train[i*self.batch_size:(i+1)*self.batch_size]
pred = sess.run(self.pred_x_filt2, feed_dict={self.x_input: next_x_images, self.keep_prob:1})
Preddiction[i*self.batch_size:(i+1)*self.batch_size] = pred.reshape([64,11])
print('Full Filtered Real Train Example Acc = ',getAcc(Preddiction, self.Y_train))
if (step == 100):
np.save('Data/x_cam3_train.npy',next_x_images)
Preddiction = np.zeros([self.TestDataSize,11])
for i in range(np.int(self.TestDataSize/self.batch_size)):
next_x_images = self.X_Real_Test[i*self.batch_size:(i+1)*self.batch_size]
pred = sess.run(self.pred_x_filt2, feed_dict={self.x_input: next_x_images, self.keep_prob:1})
Preddiction[i*self.batch_size:(i+1)*self.batch_size] = pred.reshape([64,11])
print('Full Filtered Real Cam2 Example Acc = ',getAcc(Preddiction, self.Y_test))
if (step == 100):
np.save('Data/x_cam2.npy',next_x_images)
Preddiction = np.zeros([self.TestDataSize,11])
for i in range(np.int(self.TestDataSize/self.batch_size)):
next_x_images = self.X_Real_Test[i*self.batch_size:(i+1)*self.batch_size]
pred = sess.run(self.pred_x_true, feed_dict={self.x_true: next_x_images, self.keep_prob:1})
Preddiction[i*self.batch_size:(i+1)*self.batch_size] = pred.reshape([64,11])
print('Full Real Cam2 Example Acc = ',getAcc(Preddiction, self.Y_test))
Preddiction = np.zeros([self.TestDataSize_cam3,11])
for i in range(np.int(self.TestDataSize_cam3/self.batch_size)):
next_x_images = self.X_Real_Test_cam3[i*self.batch_size:(i+1)*self.batch_size]
pred = sess.run(self.pred_x_true, feed_dict={self.x_true: next_x_images, self.keep_prob:1})
Preddiction[i*self.batch_size:(i+1)*self.batch_size] = pred.reshape([64,11])
print('Full Real Test Example Acc = ',getAcc(Preddiction, self.Y_test_cam3))
Preddiction = np.zeros([self.TrainDataSize,11])
for i in range(np.int(self.TrainDataSize/self.batch_size)):
next_x_images = self.X_Real_Train[i*self.batch_size:(i+1)*self.batch_size]
pred = sess.run(self.pred_x_true, feed_dict={self.x_true: next_x_images, self.keep_prob:1})
Preddiction[i*self.batch_size:(i+1)*self.batch_size] = pred.reshape([64,11])
print('Full Real Train Example Acc = ',getAcc(Preddiction, self.Y_train))
# print('Full Filtered Real Train Example Acc = ',getAcc(Preddiction[0:150*64], self.Y_test_cam3[0:150*64]))
# print('Full Filtered Real Test Example Acc = ',getAcc(Preddiction[150*64:], self.Y_test_cam3[150*64:]))
step += 1
def generate1(self, edge, z_var, reuse=False):
with tf.variable_scope('generator1') as scope:
if reuse == True:
scope.reuse_variables()
d1 = lrelu(fully_connect(z_var , output_size=64*4*4, scope='VAE_gen1_fully1'))
d2 = lrelu(fully_connect(d1 , output_size=128*4*4, scope='VAE_gen1_fully2'))
d3 = tf.reshape(d2, [self.batch_size, 4, 4, 128])
d4 = lrelu(de_conv(d3, output_shape=[self.batch_size, 8, 8, 128], k_h=3, k_w=3,name='VAE_gen1_deconv1'))
d5 = lrelu(de_conv(d4, output_shape=[self.batch_size, 16, 16, 128], k_h=3, k_w=3,name='VAE_gen1_deconv2'))
d6 = lrelu(de_conv(d5, output_shape=[self.batch_size, 32, 32, 128], k_h=3, k_w=3,name='VAE_gen1_deconv3'))
d7 = lrelu(de_conv(d6, output_shape=[self.batch_size, 64, 64, 128], k_h=3, k_w=3,name='VAE_gen1_deconv4'))
d8 = de_conv(d7, output_shape=[self.batch_size, 128, 128, 3] , k_h=3, k_w=3, name='VAE_gen1_deconv5')
d9 = tf.nn.tanh(d8)
d10 = tf.concat([d9, edge], 3)
conv1 = lrelu(conv2d(d10, output_dim=128, k_h=3, k_w=3, d_h=1, d_w=1,name='VAE_gen1_c1'))
conv2 = lrelu(conv2d(conv1, output_dim=128, k_h=3, k_w=3, d_h=1, d_w=1,name='VAE_gen1_c2'))
conv3 = conv2d(conv2, output_dim=3, k_h=3, k_w=3, d_h=1, d_w=1,name='VAE_gen1_c3')
return tf.nn.tanh(conv3)
def Encode1(self, x, reuse=False):
with tf.variable_scope('encode1') as scope:
if reuse == True:
scope.reuse_variables()
conv1 = lrelu(conv2d(x, output_dim=128, k_h=3, k_w=3, name='VAE_e1_c1'))
conv2 = lrelu(conv2d(conv1, output_dim=128, k_h=3, k_w=3,name='VAE_e1_c2'))
conv3 = lrelu(conv2d(conv2, output_dim=128, k_h=3, k_w=3,name='VAE_e1_c3'))
conv4 = lrelu(conv2d(conv3, output_dim=128, k_h=3, k_w=3,name='VAE_e1_c4'))
conv5 = lrelu(conv2d(conv4, output_dim=128, k_h=3, k_w=3,name='VAE_e1_c5'))
conv6 = tf.reshape(conv5, [self.batch_size, 128 * 4 * 4])
fc1 = lrelu(fully_connect(conv6, output_size= 64*4*4, scope='VAE_e1_f1'))
z_mean = fully_connect(fc1, output_size=self.latent_dim, scope='VAE_e1_f2')
z_sigma = fully_connect(fc1, output_size=self.latent_dim, scope='VAE_e1_f3')
return z_mean, z_sigma
def KL_loss(self, mu, log_var):
return -0.5 * tf.reduce_sum(1 + log_var - tf.pow(mu, 2) - tf.exp(log_var))
def sample_z(self, mu, log_var):
eps = tf.random_normal(shape=tf.shape(mu))
return mu + tf.exp(log_var / 2) * eps
def NLLNormal(self, pred, target):
c = -0.5 * tf.log(2 * np.pi)
multiplier = 1.0 / (2.0 * 1)
tmp = tf.square(pred - target)
tmp *= -multiplier
tmp += c
return tmp
def cifar10_tutorial(train_start=0, train_end=60000, test_start=0,
test_end=10000, nb_epochs=NB_EPOCHS, batch_size=BATCH_SIZE,
learning_rate=LEARNING_RATE,
clean_train=CLEAN_TRAIN,
testing=False,
backprop_through_attack=BACKPROP_THROUGH_ATTACK,
nb_filters=NB_FILTERS, num_threads=None,
label_smoothing=0.1, retrain=False,
source_samples=SOURCE_SAMPLES,
attack_iterations=ATTACK_ITERATIONS,
targeted=TARGETED):
"""
CIFAR10 cleverhans tutorial
:param train_start: index of first training set example
:param train_end: index of last training set example
:param test_start: index of first test set example
:param test_end: index of last test set example
:param nb_epochs: number of epochs to train model
:param batch_size: size of training batches
:param learning_rate: learning rate for training
:param clean_train: perform normal training on clean examples only
before performing adversarial training.
:param testing: if true, complete an AccuracyReport for unit tests
to verify that performance is adequate
:param backprop_through_attack: If True, backprop through adversarial
example construction process during
adversarial training.
:param label_smoothing: float, amount of label smoothing for cross entropy
:return: an AccuracyReport object
"""
# Object used to keep track of (and return) key accuracies
report = AccuracyReport()
# Set TF random seed to improve reproducibility
tf.set_random_seed(1234)
# Set logging level to see debug information
set_log_level(logging.DEBUG)
# Create TF session
if num_threads:
config_args = dict(intra_op_parallelism_threads=1)
else:
config_args = {}
sess = tf.Session(config=tf.ConfigProto(**config_args))
# Get CIFAR10 data
data = CIFAR10(train_start=train_start, train_end=train_end,
test_start=test_start, test_end=test_end)
dataset_size = data.x_train.shape[0]
dataset_train = data.to_tensorflow()[0]
dataset_train = dataset_train.map(
lambda x, y: (random_shift(random_horizontal_flip(x)), y), 4)
dataset_train = dataset_train.batch(batch_size)
dataset_train = dataset_train.prefetch(16)
x_train, y_train = data.get_set('train')
x_test, y_test = data.get_set('test')
###########################
# Adjust hue / saturation #
###########################
# hueValue = 0.3
# tf_x_test = tf.image.adjust_saturation(tf.image.adjust_hue(x_test, hueValue), hueValue)
# tf_x_test = tf.image.adjust_saturation(tx_test, hueValue)
# x_test = sess.run(tf_x_test)
###############################
# Transform image to uniimage #
###############################
# x_train = convert_uniimage(x_train)
# Use Image Parameters
img_rows, img_cols, nchannels = x_test.shape[1:4]
nb_classes = y_test.shape[1]
# Define input TF placeholder
x = tf.placeholder(tf.float32, shape=(None, img_rows, img_cols,
nchannels))
y = tf.placeholder(tf.float32, shape=(None, nb_classes))
saveFileNumArr = []
# saveFileNumArr = [50, 500, 1000]
count = 0
while count < 1000:
count = count + 50
saveFileNumArr.append(count)
distortionArr = []
accuracyArr = []
for i in range(len(saveFileNumArr)):
saveFileNum = saveFileNumArr[i]
model_path = os.path.join(save_dir, filename + "-" + str(saveFileNum))
# Set TF random seed to improve reproducibility
tf.set_random_seed(1234)
# Create TF session
sess = tf.Session()
print("Created TensorFlow session.")
# Define input TF placeholder
x = tf.placeholder(tf.float32, shape=(None, img_rows, img_cols,
nchannels))
y = tf.placeholder(tf.float32, shape=(None, nb_classes))
nb_filters = 64
# Define TF model graph
model = ModelAllConvolutional('model1', nb_classes, nb_filters, input_shape=[32, 32, 3])
preds = model.get_logits(x)
loss = CrossEntropy(model, smoothing=0.1)
print("Defined TensorFlow model graph.")
###########################################################################
# Training the model using TensorFlow
###########################################################################
# Train an MNIST model
train_params = {
'nb_epochs': nb_epochs,
'batch_size': batch_size,
'learning_rate': learning_rate,
'filename': os.path.split(model_path)[-1]
}
rng = np.random.RandomState([2017, 8, 30])
print("Trying to load trained model from: " + model_path)
# check if we've trained before, and if we have, use that pre-trained model
if os.path.exists(model_path + ".meta"):
tf_model_load(sess, model_path)
print("Load trained model")
else:
train(sess, loss, x_train, y_train, args=train_params, rng=rng)
saver = tf.train.Saver()
saver.save(sess, model_path)
# Evaluate the accuracy of the MNIST model on legitimate test examples
eval_params = {'batch_size': batch_size}
# accuracy = model_eval(sess, x, y, preds, x_test, y_test, args=eval_params)
# assert x_test.shape[0] == test_end - test_start, x_test.shape
# print('Test accuracy on legitimate test examples: {0}'.format(accuracy))
# report.clean_train_clean_eval = accuracy
###########################################################################
# Craft adversarial examples using Carlini and Wagner's approach
###########################################################################
nb_adv_per_sample = str(nb_classes - 1) if targeted else '1'
print('Crafting ' + str(source_samples) + ' * ' + nb_adv_per_sample +
' adversarial examples')
print("This could take some time ...")
# Instantiate a CW attack object
cw = CarliniWagnerL2(model, sess=sess)
if targeted:
adv_inputs = np.array(
[[instance] * nb_classes for
instance in x_test[:source_samples]], dtype=np.float32)
one_hot = np.zeros((nb_classes, nb_classes))
one_hot[np.arange(nb_classes), np.arange(nb_classes)] = 1
adv_inputs = adv_inputs.reshape(
(source_samples * nb_classes, img_rows, img_cols, nchannels))
adv_ys = np.array([one_hot] * source_samples,
dtype=np.float32).reshape((source_samples *
nb_classes, nb_classes))
yname = "y_target"
else:
adv_inputs = x_test[:source_samples]
adv_inputs = x_test
adv_ys = None
yname = "y"
if targeted:
cw_params_batch_size = source_samples * nb_classes
else:
cw_params_batch_size = source_samples
cw_params = {'binary_search_steps': 1,
'max_iterations': attack_iterations,
'learning_rate': CW_LEARNING_RATE,
'batch_size': cw_params_batch_size,
'initial_const': 10}
adv2 = cw.generate(x, **cw_params)
cw_params[yname] = adv_ys
adv_x = None
# adv_x = cw.generate_np(adv_inputs, **cw_params)
eval_params = {'batch_size': np.minimum(nb_classes, source_samples)}
if targeted:
accuracy = model_eval(
sess, x, y, preds, adv_x, adv_ys, args=eval_params)
else:
# err = model_eval(sess, x, y, preds, adv, y_test[:source_samples],
# args=eval_params)
accuracy, distortion = model_eval(sess, x, y, preds, x_test, y_test, args=eval_params, is_adv=True, ae=adv2,
type=type, datasetName="CIFAR10", discretizeColor=discretizeColor)
print('--------------------------------------')
print("load save file: ", saveFileNum)
# Compute the number of adversarial examples that were successfully found
# print('Test with adv. examples {0:.4f}'.format(adv_accuracy))
print('Test accuracy on examples: %0.4f ,distortion: %0.4f' % (accuracy, distortion))
distortionArr.append(distortion)
accuracyArr.append(accuracy)
# print(str(accuracy))
# print(str(distortion))
tf.reset_default_graph()
print("accuracy:")
for accuracy in accuracyArr:
print(accuracy)
print("distortion:")
for distortion in distortionArr:
print(distortion)
# Close TF session
sess.close()
return report
def cifar10_tutorial(train_start=0,
train_end=50000,
test_start=0,
test_end=10000,
nb_epochs=NB_EPOCHS,
batch_size=BATCH_SIZE,
learning_rate=LEARNING_RATE,
clean_train=CLEAN_TRAIN,
testing=False,
backprop_through_attack=BACKPROP_THROUGH_ATTACK,
nb_filters=NB_FILTERS,
num_threads=None,
label_smoothing=0.1):
"""
CIFAR10 cleverhans tutorial
:param train_start: index of first training set example
:param train_end: index of last training set example
:param test_start: index of first test set example
:param test_end: index of last test set example
:param nb_epochs: number of epochs to train model
:param batch_size: size of training batches
:param learning_rate: learning rate for training
:param clean_train: perform normal training on clean examples only
before performing adversarial training.
:param testing: if true, complete an AccuracyReport for unit tests
to verify that performance is adequate
:param backprop_through_attack: If True, backprop through adversarial
example construction process during
adversarial training.
:param label_smoothing: float, amount of label smoothing for cross entropy
:return: an AccuracyReport object
"""
# Object used to keep track of (and return) key accuracies
report = AccuracyReport()
# Set TF random seed to improve reproducibility
tf.set_random_seed(1234)
# Set logging level to see debug information
set_log_level(logging.DEBUG)
# Create TF session
if num_threads:
config_args = dict(intra_op_parallelism_threads=1)
else:
config_args = {}
sess = tf.Session(config=tf.ConfigProto(**config_args))
# Get CIFAR10 data
data = CIFAR10(train_start=train_start,
train_end=train_end,
test_start=test_start,
test_end=test_end)
dataset_size = data.x_train.shape[0]
dataset_train = data.to_tensorflow()[0]
dataset_train = dataset_train.map(
lambda x, y: (random_shift(random_horizontal_flip(x)), y), 4)
dataset_train = dataset_train.batch(batch_size)
dataset_train = dataset_train.prefetch(16)
x_train, y_train = data.get_set('train')
x_test, y_test = data.get_set('test')
# Use Image Parameters
img_rows, img_cols, nchannels = x_test.shape[1:4]
nb_classes = y_test.shape[1]
# Define input TF placeholder
x = tf.placeholder(tf.float32, shape=(None, img_rows, img_cols, nchannels))
y = tf.placeholder(tf.float32, shape=(None, nb_classes))
# Train an MNIST model
train_params = {
'nb_epochs': nb_epochs,
'batch_size': batch_size,
'learning_rate': learning_rate
}
eval_params = {'batch_size': batch_size}
fgsm_params = {'eps': 0.13, 'clip_min': 0., 'clip_max': 1.}
rng = np.random.RandomState([2017, 8, 30])
def do_eval(preds, x_set, y_set, report_key, is_adv=None):
acc = model_eval(sess, x, y, preds, x_set, y_set, args=eval_params)
setattr(report, report_key, acc)
if is_adv is None:
report_text = None
elif is_adv:
report_text = 'adversarial'
else:
report_text = 'legitimate'
if report_text:
print('Test accuracy on %s examples: %0.4f' % (report_text, acc))
model = ModelAllConvolutional('model1',
nb_classes,
nb_filters,
input_shape=[32, 32, 3])
preds = model.get_logits(x)
if clean_train:
loss = CrossEntropy(model, smoothing=label_smoothing)
def evaluate():
do_eval(preds, x_test, y_test, 'clean_train_clean_eval', False)
train(sess,
loss,
None,
None,
dataset_train=dataset_train,
dataset_size=dataset_size,
evaluate=evaluate,
args=train_params,
rng=rng,
var_list=model.get_params())
# save model
#saver = tf.train.Saver()
#saver.save(sess, "./checkpoint_dir/clean_model_100.ckpt")
# load model and compute testing accuracy
if testing:
tf_model_load(sess, file_path="./checkpoint_dir/clean_model_100.ckpt")
do_eval(preds, x_test, y_test, 'clean_train_clean_eval', False)
# Initialize the Fast Gradient Sign Method (FGSM) attack object and
# graph
fgsm = FastGradientMethod(model, sess=sess)
adv_x = fgsm.generate(x, **fgsm_params)
preds_adv = model.get_logits(adv_x)
# Evaluate the accuracy of the CIFAR10 model on adversarial examples
do_eval(preds_adv, x_test, y_test, 'clean_train_adv_eval', True)
# generate and show adversarial samples
x_test_adv = np.zeros(shape=x_test.shape)
for i in range(10):
x_test_adv[i * 1000:(i + 1) * 1000] = adv_x.eval(
session=sess, feed_dict={x: x_test[i * 1000:(i + 1) * 1000]})
# implement anisotropic diffusion on adversarial samples
x_test_filtered = np.zeros(shape=x_test_adv.shape)
for i in range(y_test.shape[0]):
x_test_filtered[i] = filter.anisotropic_diffusion(x_test_adv[i])
# implement median on adversarial samples
# x_test_filtered_med = np.zeros(shape=x_test_adv.shape)
# for i in range(y_test.shape[0]):
# x_test_filtered_med[i] = medfilt(x_test_filtered_ad[i], kernel_size=(3,3,1))
acc = model_eval(sess,
x,
y,
preds,
x_test_filtered,
y_test,
args=eval_params)
print("acc after anisotropic diffusion is {}".format(acc))
return report