def convert(graph, img, img_p, layers):
net = load_caffe(img_p, layers)
param_provider = CaffeParamProvider(net)
with tf.device('/cpu:0'):
images = tf.placeholder("float32", [None, 224, 224, 3], name="images")
m = resnet.Model(param_provider)
m.build(images, layers)
sess = tf.Session()
sess.run(tf.initialize_all_variables())
i = [
graph.get_tensor_by_name("conv1/relu:0"),
graph.get_tensor_by_name("pool1:0"),
graph.get_tensor_by_name("res2a/relu:0"),
graph.get_tensor_by_name("res2b/relu:0"),
graph.get_tensor_by_name("res2c/relu:0"),
graph.get_tensor_by_name("res3a/relu:0"),
graph.get_tensor_by_name("res5c/relu:0"),
graph.get_tensor_by_name("pool5:0"),
graph.get_tensor_by_name("prob:0"),
]
o = sess.run(i, {images: img[np.newaxis, :]})
assert_almost_equal(net.blobs['conv1'].data, o[0])
assert_almost_equal(net.blobs['pool1'].data, o[1])
assert_almost_equal(net.blobs['res2a'].data, o[2])
assert_almost_equal(net.blobs['res2b'].data, o[3])
assert_almost_equal(net.blobs['res2c'].data, o[4])
assert_almost_equal(net.blobs['res3a'].data, o[5])
assert_almost_equal(net.blobs['res5c'].data, o[6])
assert_almost_equal(net.blobs['pool5'].data, o[7])
utils.print_prob(o[8][0])
prob_dist = np.linalg.norm(net.blobs['prob'].data - o[8])
print 'prob_dist ', prob_dist
assert prob_dist < 0.2 # XXX can this be tightened?
save_graph("resnet-%d.tfmodel" % layers)
def train(config):
# seeding randomness
tf.set_random_seed(config.training.tf_random_seed)
np.random.seed(config.training.np_random_seed)
# Setting up training parameters
max_num_training_steps = config.training.max_num_training_steps
step_size_schedule = config.training.step_size_schedule
weight_decay = config.training.weight_decay
momentum = config.training.momentum
batch_size = config.training.batch_size
adversarial_training = config.training.adversarial_training
eval_during_training = config.training.eval_during_training
if eval_during_training:
num_eval_steps = config.training.num_eval_steps
# Setting up output parameters
num_output_steps = config.training.num_output_steps
num_summary_steps = config.training.num_summary_steps
num_checkpoint_steps = config.training.num_checkpoint_steps
# Setting up the data and the model
data_path = config.data.data_path
raw_cifar = cifar10_input.CIFAR10Data(data_path)
global_step = tf.contrib.framework.get_or_create_global_step()
model = resnet.Model(config.model)
# uncomment to get a list of trainable variables
# model_vars = tf.trainable_variables()
# slim.model_analyzer.analyze_vars(model_vars, print_info=True)
# Setting up the optimizer
boundaries = [int(sss[0]) for sss in step_size_schedule]
boundaries = boundaries[1:]
values = [sss[1] for sss in step_size_schedule]
learning_rate = tf.train.piecewise_constant(tf.cast(global_step, tf.int32),
boundaries, values)
total_loss = model.mean_xent + weight_decay * model.weight_decay_loss
optimizer = tf.train.MomentumOptimizer(learning_rate, momentum)
train_step = optimizer.minimize(total_loss, global_step=global_step)
# Set up adversary
attack = SpatialAttack(model, config.attack)
# Setting up the Tensorboard and checkpoint outputs
model_dir = config.model.output_dir
if eval_during_training:
eval_dir = os.path.join(model_dir, 'eval')
if not os.path.exists(eval_dir):
os.makedirs(eval_dir)
# We add accuracy and xent twice so we can easily make three types of
# comparisons in Tensorboard:
# - train vs eval (for a single run)
# - train of different runs
# - eval of different runs
saver = tf.train.Saver(max_to_keep=3)
tf.summary.scalar('accuracy_adv_train',
model.accuracy,
collections=['adv'])
tf.summary.scalar('accuracy_adv', model.accuracy, collections=['adv'])
tf.summary.scalar('xent_adv_train',
model.xent / batch_size,
collections=['adv'])
tf.summary.scalar('xent_adv', model.xent / batch_size, collections=['adv'])
tf.summary.image('images_adv_train', model.x_image, collections=['adv'])
adv_summaries = tf.summary.merge_all('adv')
tf.summary.scalar('accuracy_nat_train',
model.accuracy,
collections=['nat'])
tf.summary.scalar('accuracy_nat', model.accuracy, collections=['nat'])
tf.summary.scalar('xent_nat_train',
model.xent / batch_size,
collections=['nat'])
tf.summary.scalar('xent_nat', model.xent / batch_size, collections=['nat'])
tf.summary.image('images_nat_train', model.x_image, collections=['nat'])
tf.summary.scalar('learning_rate', learning_rate, collections=['nat'])
nat_summaries = tf.summary.merge_all('nat')
os.environ["CUDA_VISIBLE_DEVICES"] = "1"
gpu_options = tf.GPUOptions(allow_growth=True)
with tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) as sess:
# initialize data augmentation
if config.training.data_augmentation:
cifar = cifar10_input.AugmentedCIFAR10Data(raw_cifar, sess)
else:
cifar = raw_cifar
# Initialize the summary writer, global variables, and our time counter.
summary_writer = tf.summary.FileWriter(model_dir, sess.graph)
if eval_during_training:
eval_summary_writer = tf.summary.FileWriter(eval_dir)
sess.run(tf.global_variables_initializer())
training_time = 0.0
# Main training loop
for ii in range(max_num_training_steps + 1):
x_batch, y_batch = cifar.train_data.get_next_batch(
batch_size, multiple_passes=True)
noop_trans = np.zeros([len(x_batch), 3])
# Compute Adversarial Perturbations
if adversarial_training:
start = timer()
x_batch_adv, adv_trans = attack.perturb(x_batch, y_batch, sess)
end = timer()
training_time += end - start
else:
x_batch_adv, adv_trans = x_batch, noop_trans
nat_dict = {
model.x_input: x_batch,
model.y_input: y_batch,
model.transform: noop_trans,
model.is_training: False
}
adv_dict = {
model.x_input: x_batch_adv,
model.y_input: y_batch,
model.transform: adv_trans,
model.is_training: False
}
# Output to stdout
if ii % num_output_steps == 0:
nat_acc = sess.run(model.accuracy, feed_dict=nat_dict)
adv_acc = sess.run(model.accuracy, feed_dict=adv_dict)
print('Step {}: ({})'.format(ii, datetime.now()))
print(' training nat accuracy {:.4}%'.format(nat_acc * 100))
print(' training adv accuracy {:.4}%'.format(adv_acc * 100))
if ii != 0:
print(' {} examples per second'.format(
num_output_steps * batch_size / training_time))
training_time = 0.0
# Tensorboard summaries
if ii % num_summary_steps == 0:
summary = sess.run(adv_summaries, feed_dict=adv_dict)
summary_writer.add_summary(summary, global_step.eval(sess))
summary = sess.run(nat_summaries, feed_dict=nat_dict)
summary_writer.add_summary(summary, global_step.eval(sess))
# Write a checkpoint
if ii % num_checkpoint_steps == 0:
saver.save(sess,
os.path.join(model_dir, 'checkpoint'),
global_step=global_step)
if eval_during_training and ii % num_eval_steps == 0:
evaluate(model, attack, sess, config, eval_summary_writer)
# Actual training step
start = timer()
if adversarial_training:
adv_dict[model.is_training] = True
sess.run(train_step, feed_dict=adv_dict)
else:
nat_dict[model.is_training] = True
sess.run(train_step, feed_dict=nat_dict)
end = timer()
training_time += end - start
parser.add_argument('-c',
'--config',
type=str,
help='path to config file',
default="config.json",
required=False)
parser.add_argument('--loop',
help='continuously monitor model_dir'
'evaluating new ckpt',
action="store_true")
args = parser.parse_args()
config_dict = utilities.get_config(args.config)
config = utilities.config_to_namedtuple(config_dict)
model = resnet.Model(config.model)
model_dir = config.model.output_dir
global_step = tf.contrib.framework.get_or_create_global_step()
attack = SpatialAttack(model, config.attack)
if args.loop:
eval_dir = os.path.join(model_dir, 'eval')
if not os.path.exists(eval_dir):
os.makedirs(eval_dir)
summary_writer = tf.summary.FileWriter(eval_dir)
loop(model, attack, config, summary_writer)
else:
saver = tf.train.Saver()
config_dict_copy = copy.deepcopy(config_dict)
out_dict = {}
out_dict['hyperparameters'] = config_dict_copy
config = utilities.config_to_namedtuple(config_dict)
reg_model_path = args.reg_model_path
# num_ids in model does not matter for eval
num_ids = 64
model_family = config.model.model_family
if model_family == "resnet":
if config.attack.use_spatial and config.attack.spatial_method == 'fo':
diffable = True
else:
diffable = False
model = resnet.Model(config.model, num_ids, diffable)
elif model_family == "vgg":
if config.attack.use_spatial and config.attack.spatial_method == 'fo':
diffable = True
else:
diffable = False
model = vgg.Model(config.model, num_ids, diffable)
global_step = tf.train.get_or_create_global_step()
if args.linf_attack:
attack_eval = SpatialAttack(model, config.attack, 'fo', 1,
config.attack.spatial_limits,
config.attack.epsilon,
config.attack.step_size,
config.attack.num_steps)
else:
def train(config='configs/fannyconfig.json',
save_root_path='/cluster/work/math/fanyang-broglil/CoreRepo',
experiment_json_fname='experiments.json',
local_json_dir_name='local_json_files',
worstofk=None,
attack_style=None,
attack_limits=None,
fo_epsilon=None,
fo_step_size=None,
fo_num_steps=None,
lambda_core=None,
num_ids=None,
group_size=None,
use_core=None,
seed=None,
save_in_local_json=True,
this_repo=None):
# reset default graph (needed for running locally with run_jobs_ray.py)
tf.reset_default_graph()
# get configs
config_dict = utilities.get_config(config)
config_dict_copy = copy.deepcopy(config_dict)
config = utilities.config_to_namedtuple(config_dict)
# seeding randomness
if seed == None:
seed = config.training.tf_random_seed
else:
config_dict_copy['training']['tf_random_seed'] = seed
tf.set_random_seed(seed)
np.random.seed(seed)
# Setting up training parameters
max_num_training_steps = config.training.max_num_training_steps
step_size_schedule = config.training.step_size_schedule
weight_decay = config.training.weight_decay
momentum = config.training.momentum
if group_size == None:
group_size = config.training.group_size
else:
config_dict_copy['training']['group_size'] = int(group_size)
if num_ids == None:
num_ids = config.training.num_ids
else:
config_dict_copy['training']['num_ids'] = int(num_ids)
if lambda_core == None:
lambda_core = config.training.lambda_
else:
config_dict_copy['training']['lambda_'] = float(lambda_core)
if use_core == None:
use_core = config.training.use_core
else:
config_dict_copy['training']['use_core'] = use_core
batch_size = config.training.batch_size
# number of groups with group size > 1
num_grouped_ids = batch_size - num_ids
# number of unique ids needs to be larger than half the desired batch size
# so that full batch can be filled up
assert num_ids >= batch_size / group_size
# currently, code is designed for groups of size 2
# assert batch_size % group_size == 0
adversarial_training = config.training.adversarial_training
eval_during_training = config.training.eval_during_training
if eval_during_training:
num_eval_steps = config.training.num_eval_steps
# Setting up output parameters
num_output_steps = config.training.num_output_steps
num_summary_steps = config.training.num_summary_steps
num_checkpoint_steps = config.training.num_checkpoint_steps
num_easyeval_steps = config.training.num_easyeval_steps
# Setting up the data and the model
data_path = config.data.data_path
if config.data.dataset_name == "cifar-10":
raw_iterator = cifar10_input.CIFAR10Data(data_path)
elif config.data.dataset_name == "cifar-100":
raw_iterator = cifar100_input.CIFAR100Data(data_path)
elif config.data.dataset_name == "svhn":
raw_iterator = svhn_input.SVHNData(data_path)
else:
raise ValueError("Unknown dataset name.")
global_step = tf.train.get_or_create_global_step()
model_family = config.model.model_family
if model_family == "resnet":
if config.attack.use_spatial and config.attack.spatial_method == 'fo':
diffable = True
else:
diffable = False
model = resnet.Model(config.model, num_ids, diffable)
elif model_family == "vgg":
if config.attack.use_spatial and config.attack.spatial_method == 'fo':
# TODO: add differentiable transformer to vgg.py
raise NotImplementedError
model = vgg.Model(config.model, num_ids)
# uncomment to get a list of trainable variables
# model_vars = tf.trainable_variables()
# Setting up the optimizer
boundaries = [int(sss[0]) for sss in step_size_schedule]
boundaries = boundaries[1:]
values = [sss[1] for sss in step_size_schedule]
learning_rate = tf.train.piecewise_constant(tf.cast(global_step, tf.int32),
boundaries, values)
if use_core and lambda_core > 0:
total_loss = (model.mean_xent +
weight_decay * model.weight_decay_loss +
lambda_core * model.core_loss)
else:
total_loss = model.mean_xent + weight_decay * model.weight_decay_loss
optimizer = tf.train.MomentumOptimizer(learning_rate, momentum)
train_step = optimizer.minimize(total_loss, global_step=global_step)
# Set up adversary
if worstofk == None:
worstofk = config.attack.random_tries
else:
config_dict_copy['attack']['random_tries'] = worstofk
if fo_epsilon == None:
fo_epsilon = config.attack.epsilon
else:
config_dict_copy['attack']['epsilon'] = fo_epsilon
if fo_step_size == None:
fo_step_size = config.attack.step_size
else:
config_dict_copy['attack']['step_size'] = fo_step_size
if fo_num_steps == None:
fo_num_steps = config.attack.num_steps
else:
config_dict_copy['attack']['num_steps'] = fo_num_steps
# @ Luzius: incorporate being able to choose multiple transformations
if attack_style == None:
attack_style = 'rotate'
# Training attack
# L-inf attack if use_spatial is False and use_linf is True
# spatial attack if use_spatial is True and use_linf is False
# spatial random attack if spatial_method is 'random'
# spatial PGD attack if spatial_method is 'fo'
attack = SpatialAttack(model, config.attack, config.attack.spatial_method,
worstofk, attack_limits, fo_epsilon, fo_step_size,
fo_num_steps)
# Different eval attacks
# Random attack
# L-inf attack if use_spatial is False and use_linf is True
# random (worst-of-1) spatial attack if use_spatial is True
# and use_linf is False
attack_eval_random = SpatialAttack(model, config.attack, 'random', 1,
attack_limits, fo_epsilon, fo_step_size,
fo_num_steps)
# First order attack
# L-inf attack if use_spatial is False and use_linf is True
# first-order spatial attack if use_spatial is True and use_linf is False
attack_eval_fo = SpatialAttack(model, config.attack, 'fo', 1,
attack_limits, fo_epsilon, fo_step_size,
fo_num_steps)
# Grid attack
# spatial attack if use_spatial is True and use_linf is False
# not executed for L-inf attacks
attack_eval_grid = SpatialAttack(model, config.attack, 'grid', None,
attack_limits)
# TODO(christina): add L-inf attack with random restarts
# ------------------START EXPERIMENT -------------------------
# Initialize the Repo
print("==> Creating repo..")
# Create repo object if it wasn't passed, comment out if repo has issues
if this_repo == None:
this_repo = exprepo.ExperimentRepo(
save_in_local_json=save_in_local_json,
json_filename=experiment_json_fname,
local_dir_name=local_json_dir_name,
root_dir=save_root_path)
# Create new experiment
if this_repo != None:
exp_id = this_repo.create_new_experiment(config.data.dataset_name,
model_family, worstofk,
attack_style, attack_limits,
lambda_core, num_grouped_ids,
group_size, config_dict_copy)
# Setting up the Tensorboard and checkpoint outputs
model_dir = '%s/logdir/%s' % (save_root_path, exp_id)
# We add accuracy and xent twice so we can easily make three types of
# comparisons in Tensorboard:
# - train vs eval (for a single run)
# - train of different runs
# - eval of different runs
saver = tf.train.Saver(max_to_keep=3)
tf.summary.scalar('accuracy_nat_train',
model.accuracy,
collections=['nat'])
tf.summary.scalar('accuracy_nat', model.accuracy, collections=['nat'])
tf.summary.scalar('xent_nat_train',
model.xent / batch_size,
collections=['nat'])
tf.summary.scalar('xent_nat', model.xent / batch_size, collections=['nat'])
tf.summary.image('images_nat_train', model.x_image, collections=['nat'])
tf.summary.scalar('learning_rate', learning_rate, collections=['nat'])
nat_summaries = tf.summary.merge_all('nat')
# data augmentation used if config.training.data_augmentation_core is True
x_input_placeholder = tf.placeholder(tf.float32, shape=[None, 32, 32, 3])
flipped = tf.map_fn(lambda img: tf.image.random_flip_left_right(img),
x_input_placeholder)
with tf.Session() as sess:
# initialize standard data augmentation
if config.training.data_augmentation:
if config.data.dataset_name == "cifar-10":
data_iterator = cifar10_input.AugmentedCIFAR10Data(
raw_iterator, sess)
elif config.data.dataset_name == "cifar-100":
data_iterator = cifar100_input.AugmentedCIFAR100Data(
raw_iterator, sess)
elif config.data.dataset_name == "svhn":
data_iterator = svhn_input.AugmentedSVHNData(
raw_iterator, sess)
else:
raise ValueError("Unknown dataset name.")
else:
data_iterator = raw_iterator
eval_dict = {
model.x_input: data_iterator.eval_data.xs,
model.y_input: data_iterator.eval_data.ys,
model.group: np.arange(0, batch_size, 1, dtype="int32"),
model.transform: np.zeros([data_iterator.eval_data.n, 3]),
model.is_training: False
}
# Initialize the summary writer, global variables, and our time counter.
summary_writer = tf.summary.FileWriter(model_dir, sess.graph)
# if eval_during_training:
eval_dir = os.path.join(model_dir, 'eval')
os.makedirs(eval_dir, exist_ok=True)
eval_summary_writer = tf.summary.FileWriter(eval_dir)
sess.run(tf.global_variables_initializer())
training_time = 0.0
run_time_without_eval = 0.0
run_time_adv_ex_creation = 0.0
run_time_train_step = 0.0
####################################
# Main training loop
####################################
start_time = time.time()
no_epochs_done = 0 # the same as epoch_count, need to merge
start_epoch = timer()
it_count = 0
epoch_count = 0
acc_sum = 0
for ii in range(max_num_training_steps + 1):
# original batch
x_batch, y_batch, epoch_done = data_iterator.train_data.get_next_batch(
num_ids, multiple_passes=True)
no_epochs_done += epoch_done
# noop trans
noop_trans = np.zeros([len(x_batch), 3])
# id_batch starts with IDs of original examples
id_batch = np.arange(0, num_ids, 1, dtype="int32")
if use_core:
# first num_id examples of batch are natural
x_batch_inp = x_batch
y_batch_inp = y_batch
trans_inp = noop_trans
id_batch_inp = id_batch
start = timer()
for _ in range(group_size - 1):
if config.training.data_augmentation_core:
raise NotImplementedError
# create rotated examples
x_batch_adv_i, trans_adv_i = attack.perturb(
x_batch, y_batch, sess)
# construct new batches including rotated examples
x_batch_inp = np.concatenate((x_batch_inp, x_batch_adv_i),
axis=0)
y_batch_inp = np.concatenate((y_batch_inp, y_batch),
axis=0)
trans_inp = np.concatenate((trans_inp, trans_adv_i),
axis=0)
id_batch_inp = np.concatenate((id_batch_inp, id_batch),
axis=0)
end = timer()
training_time += end - start
run_time_without_eval += end - start
run_time_adv_ex_creation += end - start
trans_adv = trans_inp[num_ids:, ...]
id_batch_adv = id_batch_inp[num_ids:]
y_batch_adv = y_batch_inp[num_ids:]
x_batch_adv = x_batch_inp[num_ids:, ...]
else:
if adversarial_training:
start = timer()
x_batch_inp, trans_inp = attack.perturb(
x_batch, y_batch, sess)
end = timer()
training_time += end - start
run_time_without_eval += end - start
run_time_adv_ex_creation += end - start
else:
x_batch_inp, trans_inp = x_batch, noop_trans
# for adversarial training and plain training, the following
# variables coincide
y_batch_inp = y_batch
y_batch_adv = y_batch
trans_adv = trans_inp
x_batch_adv = x_batch_inp
id_batch_inp = id_batch
id_batch_adv = id_batch
# feed_dict for training step
inp_dict = {
model.x_input: x_batch_inp,
model.y_input: y_batch_inp,
model.group: id_batch_inp,
model.transform: trans_inp,
model.is_training: False
}
# separate natural and adversarially transformed examples for eval
nat_dict = {
model.x_input: x_batch,
model.y_input: y_batch,
model.group: id_batch,
model.transform: noop_trans,
model.is_training: False
}
adv_dict = {
model.x_input: x_batch_adv,
model.y_input: y_batch_adv,
model.group: id_batch_adv,
model.transform: trans_adv,
model.is_training: False
}
########### Outputting/saving weights and evaluations ###########
acc_grid_te = -1.0
avg_xent_grid_te = -1.0
acc_fo_te = -1.0
avg_xent_fo_te = -1.0
saved_weights = 0
# Compute training accuracy on this minibatch
nat_acc_tr = 100 * sess.run(model.accuracy, feed_dict=nat_dict)
# Output to stdout
if epoch_done:
epoch_time = timer() - start_epoch
# Average
av_acc = acc_sum / it_count
# ToDo: Log this to file as well
# Training accuracy over epoch
print('Epoch {}: ({})'.format(epoch_count, datetime.now()))
print(' training natural accuracy {:.4}%'.format(av_acc))
print(' {:.4} seconds per epoch'.format(epoch_time))
# Accuracy on entire test set
nat_acc_te = 100 * sess.run(model.accuracy,
feed_dict=eval_dict)
print(
' test set natural accuracy {:.4}%'.format(nat_acc_te))
# Set loss sum, it count back to zero
acc_sum = nat_acc_tr
epoch_done = 0
epoch_count += 1
start_epoch = timer()
it_count = 1
else:
it_count += 1
acc_sum += nat_acc_tr
# Output to stdout
if ii % num_output_steps == 0:
# nat_acc_tr = 100 * sess.run(model.accuracy, feed_dict=nat_dict)
adv_acc_tr = 100 * sess.run(model.accuracy, feed_dict=adv_dict)
inp_acc_tr = 100 * sess.run(model.accuracy, feed_dict=inp_dict)
# print('Step {}: ({})'.format(ii, datetime.now()))
# print(' training nat accuracy {:.4}%'.format(nat_acc_tr))
# print(' training adv accuracy {:.4}%'.format(adv_acc_tr))
# print(' training inp accuracy {:.4}%'.format(inp_acc_tr))
if ii != 0:
# print(' {} examples per second'.format(
# num_output_steps * batch_size / training_time))
training_time = 0.0
# Tensorboard summaries and heavy checkpoints
if ii % num_summary_steps == 0:
summary = sess.run(nat_summaries, feed_dict=nat_dict)
summary_writer.add_summary(summary, global_step.eval(sess))
# Write a checkpoint and eval if it's time
if ii % num_checkpoint_steps == 0 or ii == max_num_training_steps:
# Save checkpoint data (weights)
saver.save(sess,
os.path.join(model_dir, 'checkpoint'),
global_step=global_step)
saved_weights = 1
# Write evaluation meta data for checkpoint
if ii % num_easyeval_steps == 0 or ii == max_num_training_steps:
# Get training accuracies
nat_acc_tr = 100 * sess.run(model.accuracy, feed_dict=nat_dict)
adv_acc_tr = 100 * sess.run(model.accuracy, feed_dict=adv_dict)
inp_acc_tr = 100 * sess.run(model.accuracy, feed_dict=inp_dict)
# Evaluation on random and natural
[
acc_nat_te, acc_rand_adv_te, avg_xent_nat_te,
avg_xent_adv_te
] = evaluate(model, attack_eval_random, sess, config, 'random',
data_path, None)
# Evaluation on grid (only for spatial attacks)
if ((eval_during_training and ii % num_eval_steps == 0
and ii > 0 and config.attack.use_spatial) or
(eval_during_training and ii == max_num_training_steps
and config.attack.use_spatial)):
if config.attack.use_spatial and config.attack.spatial_method == 'fo':
# Evaluation on first-order PDG attack (too expensive to
# evaluate more frequently on whole dataset)
[_, acc_fo_te, _,
avg_xent_fo_te] = evaluate(model, attack_eval_fo,
sess, config, 'fo',
data_path, None)
# Evaluation on grid
[_, acc_grid_te, _, avg_xent_grid_te
] = evaluate(model, attack_eval_grid, sess, config,
"grid", data_path, eval_summary_writer)
chkpt_id = this_repo.create_training_checkpoint(
exp_id,
training_step=ii,
epoch=no_epochs_done,
train_acc_nat=nat_acc_tr,
train_acc_adv=adv_acc_tr,
train_acc_inp=inp_acc_tr,
test_acc_nat=acc_nat_te,
test_acc_adv=acc_rand_adv_te,
test_acc_fo=acc_fo_te,
test_acc_grid=acc_grid_te,
test_loss_nat=avg_xent_nat_te,
test_loss_adv=avg_xent_adv_te,
test_loss_fo=avg_xent_fo_te,
test_loss_grid=avg_xent_grid_te)
if saved_weights == 0:
# Save checkpoint data (weights)
saver.save(
sess,
os.path.join(model_dir,
'{}_checkpoint'.format(chkpt_id)))
# Actual training step
start = timer()
inp_dict[model.is_training] = True
sess.run(train_step, feed_dict=inp_dict)
end = timer()
training_time += end - start
run_time_without_eval += end - start
run_time_train_step += end - start
runtime = time.time() - start_time
this_repo.mark_experiment_as_completed(
exp_id,
train_acc_nat=nat_acc_tr,
train_acc_adv=adv_acc_tr,
train_acc_inp=inp_acc_tr,
test_acc_nat=acc_nat_te,
test_acc_adv=acc_rand_adv_te,
test_acc_fo=acc_fo_te,
test_acc_grid=acc_grid_te,
runtime=runtime,
runtime_wo_eval=run_time_without_eval,
runtime_train_step=run_time_train_step,
runtime_adv_ex_creation=run_time_adv_ex_creation)
return 0
def train(config):
# seeding randomness
tf.set_random_seed(config.training.tf_random_seed)
np.random.seed(config.training.np_random_seed)
# Setting up training parameters
max_num_training_steps = config.training.max_num_training_steps
step_size_schedule = config.training.step_size_schedule
weight_decay = config.training.weight_decay
momentum = config.training.momentum
batch_size = config.training.batch_size
eval_during_training = config.training.eval_during_training
num_clean_examples = config.training.num_examples
if eval_during_training:
num_eval_steps = config.training.num_eval_steps
# Setting up output parameters
num_output_steps = config.training.num_output_steps
num_summary_steps = config.training.num_summary_steps
num_checkpoint_steps = config.training.num_checkpoint_steps
# Setting up the data and the model
dataset = dataset_input.CIFAR10Data(config,
seed=config.training.np_random_seed)
print('Num Poisoned Left: {}'.format(dataset.num_poisoned_left))
print('Poison Position: {}'.format(config.data.position))
print('Poison Color: {}'.format(config.data.color))
num_training_examples = len(dataset.train_data.xs)
global_step = tf.contrib.framework.get_or_create_global_step()
model = resnet.Model(config.model)
# uncomment to get a list of trainable variables
model_vars = tf.trainable_variables()
slim.model_analyzer.analyze_vars(model_vars, print_info=True)
# Setting up the optimizer
boundaries = [int(sss[0]) for sss in step_size_schedule]
boundaries = boundaries[1:]
values = [sss[1] for sss in step_size_schedule]
learning_rate = tf.train.piecewise_constant(tf.cast(global_step, tf.int32),
boundaries, values)
total_loss = model.mean_xent + weight_decay * model.weight_decay_loss
optimizer = tf.train.MomentumOptimizer(learning_rate, momentum)
train_step = optimizer.minimize(total_loss, global_step=global_step)
# Setting up the Tensorboard and checkpoint outputs
model_dir = config.model.output_dir
if eval_during_training:
eval_dir = os.path.join(model_dir, 'eval')
if not os.path.exists(eval_dir):
os.makedirs(eval_dir)
# We add accuracy and xent twice so we can easily make three types of
# comparisons in Tensorboard:
# - train vs eval (for a single run)
# - train of different runs
# - eval of different runs
saver = tf.train.Saver(max_to_keep=3)
tf.summary.scalar('accuracy_nat_train',
model.accuracy,
collections=['nat'])
tf.summary.scalar('accuracy_nat', model.accuracy, collections=['nat'])
tf.summary.scalar('xent_nat_train',
model.xent / batch_size,
collections=['nat'])
tf.summary.scalar('xent_nat', model.xent / batch_size, collections=['nat'])
tf.summary.image('images_nat_train', model.train_xs, collections=['nat'])
tf.summary.scalar('learning_rate', learning_rate, collections=['nat'])
nat_summaries = tf.summary.merge_all('nat')
with tf.Session() as sess:
print('Dataset Size: ', len(dataset.train_data.xs))
# Initialize the summary writer, global variables, and our time counter.
summary_writer = tf.summary.FileWriter(model_dir, sess.graph)
if eval_during_training:
eval_summary_writer = tf.summary.FileWriter(eval_dir)
sess.run(tf.global_variables_initializer())
training_time = 0.0
# Main training loop
for ii in range(max_num_training_steps + 1):
x_batch, y_batch = dataset.train_data.get_next_batch(
batch_size, multiple_passes=True)
nat_dict = {
model.x_input: x_batch,
model.y_input: y_batch,
model.is_training: False
}
# Output to stdout
if ii % num_output_steps == 0:
nat_acc = sess.run(model.accuracy, feed_dict=nat_dict)
print('Step {}: ({})'.format(ii, datetime.now()))
print(' training nat accuracy {:.4}%'.format(nat_acc * 100))
if ii != 0:
print(' {} examples per second'.format(
num_output_steps * batch_size / training_time))
training_time = 0.0
# Tensorboard summaries
if ii % num_summary_steps == 0:
summary = sess.run(nat_summaries, feed_dict=nat_dict)
summary_writer.add_summary(summary, global_step.eval(sess))
# Write a checkpoint
if ii % num_checkpoint_steps == 0:
saver.save(sess,
os.path.join(model_dir, 'checkpoint'),
global_step=global_step)
if eval_during_training and ii % num_eval_steps == 0:
evaluate(model, sess, config, eval_summary_writer)
# Actual training step
start = timer()
nat_dict[model.is_training] = True
sess.run(train_step, feed_dict=nat_dict)
end = timer()
training_time += end - start
def train(config):
# seeding randomness
tf.set_random_seed(config.training.tf_random_seed)
np.random.seed(config.training.np_random_seed)
# Setting up training parameters
max_num_training_steps = config.training.max_num_training_steps
step_size_schedule = config.training.step_size_schedule
weight_decay = config.training.weight_decay
momentum = config.training.momentum
batch_size = config.training.batch_size
group_size = config.training.group_size
adversarial_training = config.training.adversarial_training
eval_during_training = config.training.eval_during_training
if eval_during_training:
num_eval_steps = config.training.num_eval_steps
# Setting up output parameters
num_output_steps = config.training.num_output_steps
num_summary_steps = config.training.num_summary_steps
num_checkpoint_steps = config.training.num_checkpoint_steps
#adapting batch size
batch_size_group = batch_size * config.training.group_size
# Setting up the data and the model
data_path = config.data.data_path
raw_cifar = cifar10_input.CIFAR10Data(data_path)
global_step = tf.train.get_or_create_global_step()
model = resnet.Model(config.model)
# uncomment to get a list of trainable variables
# model_vars = tf.trainable_variables()
# slim.model_analyzer.analyze_vars(model_vars, print_info=True)
# Setting up the optimizer
boundaries = [int(sss[0]) for sss in step_size_schedule]
boundaries = boundaries[1:]
values = [sss[1] for sss in step_size_schedule]
learning_rate = tf.train.piecewise_constant(tf.cast(global_step, tf.int32),
boundaries, values)
total_loss = model.mean_xent + weight_decay * model.weight_decay_loss + model.core_loss
optimizer = tf.train.MomentumOptimizer(learning_rate, momentum)
train_step = optimizer.minimize(total_loss, global_step=global_step)
# Set up adversary
attack = SpatialAttack(model, config.attack)
attack_eval_random = SpatialAttack(model, config.eval_attack_random)
attack_eval_grid = SpatialAttack(model, config.eval_attack_grid)
# Setting up the Tensorboard and checkpoint outputs
model_dir = config.model.output_dir
if eval_during_training:
eval_dir = os.path.join(model_dir, 'eval')
if not os.path.exists(eval_dir):
os.makedirs(eval_dir)
# We add accuracy and xent twice so we can easily make three types of
# comparisons in Tensorboard:
# - train vs eval (for a single run)
# - train of different runs
# - eval of different runs
saver = tf.train.Saver(max_to_keep=3)
tf.summary.scalar('accuracy_adv_train',
model.accuracy,
collections=['adv'])
tf.summary.scalar('accuracy_adv', model.accuracy, collections=['adv'])
tf.summary.scalar('xent_adv_train',
model.xent / batch_size_group,
collections=['adv'])
tf.summary.scalar('xent_adv',
model.xent / batch_size_group,
collections=['adv'])
tf.summary.image('images_adv_train', model.x_image, collections=['adv'])
adv_summaries = tf.summary.merge_all('adv')
tf.summary.scalar('accuracy_nat_train',
model.accuracy,
collections=['nat'])
tf.summary.scalar('accuracy_nat', model.accuracy, collections=['nat'])
tf.summary.scalar('xent_nat_train',
model.xent / batch_size_group,
collections=['nat'])
tf.summary.scalar('xent_nat',
model.xent / batch_size_group,
collections=['nat'])
tf.summary.image('images_nat_train', model.x_image, collections=['nat'])
tf.summary.scalar('learning_rate', learning_rate, collections=['nat'])
nat_summaries = tf.summary.merge_all('nat')
#dataAugmentation
x_input_placeholder = tf.placeholder(tf.float32, shape=[None, 32, 32, 3])
flipped = tf.map_fn(lambda img: tf.image.random_flip_left_right(img),
x_input_placeholder)
with tf.Session() as sess:
# initialize data augmentation
if config.training.data_augmentation:
cifar = cifar10_input.AugmentedCIFAR10Data(raw_cifar, sess)
else:
cifar = raw_cifar
# Initialize the summary writer, global variables, and our time counter.
summary_writer = tf.summary.FileWriter(model_dir, sess.graph)
if eval_during_training:
eval_summary_writer = tf.summary.FileWriter(eval_dir)
sess.run(tf.global_variables_initializer())
training_time = 0.0
training_time_total = 0.0
adv_time = 0.0
eval_time = 0.0
core_time = 0.0
# Main training loop
for ii in range(max_num_training_steps + 1):
x_batch, y_batch = cifar.train_data.get_next_batch(
batch_size, multiple_passes=True)
noop_trans = np.zeros([len(x_batch), 3])
# Compute Adversarial Perturbations
if adversarial_training:
start = timer()
x_batch_adv, adv_trans = attack.perturb(x_batch, y_batch, sess)
end = timer()
adv_time += end - start
else:
x_batch_adv, adv_trans = x_batch, noop_trans
#Create rotatated examples
start = timer()
x_batch_nat = x_batch
y_batch_nat = y_batch
id_batch = np.arange(0, batch_size, 1, dtype="int32")
ids = np.arange(0, batch_size, 1, dtype="int32")
for i in range(config.training.group_size):
if config.training.data_augmentation_core:
x_batch_core = sess.run(
flipped, feed_dict={x_input_placeholder: x_batch})
else:
x_batch_core = x_batch
x_batch_group, trans_group = attack.perturb(
x_batch_core, y_batch, sess)
#construct new batches including rotateted examples
x_batch_adv = np.concatenate((x_batch_adv, x_batch_group),
axis=0)
x_batch_nat = np.concatenate((x_batch_nat, x_batch_group),
axis=0)
y_batch_nat = np.concatenate((y_batch_nat, y_batch), axis=0)
adv_trans = np.concatenate((adv_trans, trans_group), axis=0)
noop_trans = np.concatenate((noop_trans, trans_group), axis=0)
id_batch = np.concatenate((id_batch, ids), axis=0)
end = timer()
core_time += end - start
nat_dict = {
model.x_input: x_batch_nat,
model.y_input: y_batch_nat,
model.group: id_batch,
model.transform: noop_trans,
model.is_training: False
}
adv_dict = {
model.x_input: x_batch_adv,
model.y_input: y_batch_nat,
model.group: id_batch,
model.transform: adv_trans,
model.is_training: False
}
# Output to stdout
if ii % num_output_steps == 0:
nat_acc = sess.run(model.accuracy, feed_dict=nat_dict)
adv_acc = sess.run(model.accuracy, feed_dict=adv_dict)
print('Step {}: ({})'.format(ii, datetime.now()))
print(' training nat accuracy {:.4}%'.format(nat_acc * 100))
print(' training adv accuracy {:.4}%'.format(adv_acc * 100))
if ii != 0:
print(' {} examples per second'.format(
num_output_steps * batch_size_group / training_time))
training_time = 0.0
# Tensorboard summaries
if ii % num_summary_steps == 0:
summary = sess.run(adv_summaries, feed_dict=adv_dict)
summary_writer.add_summary(summary, global_step.eval(sess))
summary = sess.run(nat_summaries, feed_dict=nat_dict)
summary_writer.add_summary(summary, global_step.eval(sess))
# Write a checkpoint
if ii % num_checkpoint_steps == 0:
saver.save(sess,
os.path.join(model_dir, 'checkpoint'),
global_step=global_step)
if eval_during_training and ii % num_eval_steps == 0:
start = timer()
evaluate(model, attack_eval_random, sess, config, "random",
eval_summary_writer)
evaluate(model, attack_eval_grid, sess, config, "grid",
eval_summary_writer)
end = timer()
eval_time += end - start
print(' {}seconds total training time'.format(
training_time_total))
print(' {}seconds total adv. example time'.format(adv_time))
print(
' {}seconds total core example time'.format(core_time))
print(' {}seconds total evalutation time'.format(eval_time))
# Actual training step
start = timer()
if adversarial_training:
adv_dict[model.is_training] = True
sess.run(train_step, feed_dict=adv_dict)
else:
nat_dict[model.is_training] = True
sess.run(train_step, feed_dict=nat_dict)
end = timer()
training_time += end - start
training_time_total += end - start
# ap = argparse.ArgumentParser()
# ap.add_argument("-train", "--train", type=bool, default = True,
# help="Training set")
# ap.add_argument("-test", "--test", type=bool, default=False,
# help="Testing set")
# args = vars(ap.parse_args())
# # grab the number of GPUs and store it in a conveience variable
# train = args["train"]
# test = args["test"]
train = args.train
test = args.test
pred = args.predict
input_img = Input((im_height, im_width, 1), name='img')
model = import resnet.Model(input_img)
# save_path=cwd + '/weights/weight.h5'
# loss = "binary_crossentropy"
# Train
if train:
train_model(model, save_path=save_path, loss=loss)
# Test
if test:
weight_path = save_path
threshold = 0.5
test_model(weight_path, threshold)
if pred:
def __init__(self, sess, config, filepath, batch_size, regu,
learning_rate = 0.1,
binary_search_steps = 1, max_iterations = 101,
initial_const = 1):
image_size, num_channels, num_labels = 32, 3, 10
self.sess = sess
self.LEARNING_RATE = learning_rate
self.MAX_ITERATIONS = max_iterations
self.BINARY_SEARCH_STEPS = binary_search_steps
#self.ABORT_EARLY = abort_early
self.initial_const = initial_const
self.batch_size = batch_size
shape = (batch_size,image_size,image_size,num_channels)
shape_pert = (batch_size,image_size,image_size,num_channels)
# these are variables to be more efficient in sending data to tf
self.timg = tf.Variable(np.zeros(shape), dtype=tf.float32)
self.tlab = tf.Variable(np.zeros((batch_size,num_labels)), dtype=tf.float32)
self.const = tf.Variable(np.zeros(batch_size), dtype=tf.float32)
self.modifier = tf.Variable(np.zeros(shape_pert,dtype=np.float32))
self.det = tf.Variable(np.ones(shape_pert,dtype=np.float32), constraint = lambda x:tf.clip_by_value(x, 0, 255))
self.newimg = self.modifier*self.det + self.timg*(1-self.modifier)
self.assign_tlab = tf.placeholder(tf.float32, (batch_size,num_labels), name="tlab")
self.assign_timg = tf.placeholder(tf.float32, shape, name="timag")
self.assign_const = tf.placeholder(tf.float32, [batch_size], name="tconst")
# the variable we're going to optimize over
global_step = tf.contrib.framework.get_or_create_global_step()
#with tf.variable_scope(reuse=tf.AUTO_REUSE):
model = resnet.Model(config.model, self.newimg)
# Setting up the Tensorboard and checkpoint outputs
model_dir = filepath
#saver = tf.train.Saver()
var_list = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES)
var_list = var_list[5:]
#saver = tf.train.Saver(var_list=[v for v in all_variables if not in v.name])
saver = tf.train.Saver(max_to_keep=3, var_list=var_list)
latest_checkpoint = tf.train.latest_checkpoint(model_dir)
if latest_checkpoint is not None:
saver.restore(sess, latest_checkpoint)
print('Restoring last saved checkpoint: ', latest_checkpoint)
else:
print('Check model directory')
exit()
self.output=model.pre_softmax
# compute the probability of the label class versus the maximum other
real = tf.reduce_sum((self.tlab)*self.output,1)
other = tf.reduce_max((1-self.tlab)*self.output - (self.tlab*10000),1)
loss1 = tf.maximum(-15.0, other - real)
self.loss1 = tf.reduce_sum(loss1)
if regu == "l2":
self.loss2 = tf.reduce_sum(tf.square(self.modifier))
else:
self.loss2 = tf.reduce_sum(tf.abs(self.modifier))
self.loss = self.loss1
self.modifier_ST = tf.clip_by_value(tf.sign(self.modifier)*tf.maximum(tf.abs(self.modifier) - self.LEARNING_RATE/self.initial_const, 0), clip_value_min=0, clip_value_max=1)
self.assign_ST = tf.assign(self.modifier, self.modifier_ST)
# Setup the adam optimizer and keep track of variables we're creating
start_vars = set(x.name for x in tf.global_variables())
optimizer = tf.train.AdamOptimizer(self.LEARNING_RATE)
self.train1= optimizer.minimize(self.loss, var_list=[self.modifier])
self.train2 = optimizer.minimize(self.loss, var_list=[self.det])
end_vars = tf.global_variables()
new_vars = [x for x in end_vars if x.name not in start_vars]
# these are the variables to initialize when we run
self.setup = []
self.setup.append(self.timg.assign(self.assign_timg))
self.setup.append(self.tlab.assign(self.assign_tlab))
self.setup.append(self.const.assign(self.assign_const))
#self.init = tf.variables_initializer(var_list=[self.modifier]+new_vars)
self.init = tf.variables_initializer(var_list=[self.modifier]+[self.det]+new_vars)
def compute_corr(config):
# seeding randomness
tf.set_random_seed(config.training.tf_random_seed)
np.random.seed(config.training.np_random_seed)
# Setting up the data and the model
poison_eps = config.data.poison_eps
clean_label = config.data.clean_label
target_label = config.data.target_label
dataset = dataset_input.CIFAR10Data(config,
seed=config.training.np_random_seed)
num_poisoned_left = dataset.num_poisoned_left
print('Num poisoned left: ', num_poisoned_left)
num_training_examples = len(dataset.train_data.xs)
global_step = tf.contrib.framework.get_or_create_global_step()
model = resnet.Model(config.model)
# Setting up the Tensorboard and checkpoint outputs
model_dir = config.model.output_dir
saver = tf.train.Saver(max_to_keep=3)
with tf.Session() as sess:
# initialize data augmentation
print('Dataset Size: ', len(dataset.train_data.xs))
sess.run(tf.global_variables_initializer())
latest_checkpoint = tf.train.latest_checkpoint(model_dir)
if latest_checkpoint is not None:
saver.restore(sess, latest_checkpoint)
print('Restoring last saved checkpoint: ', latest_checkpoint)
else:
print('Check model directory')
exit()
lbl = target_label
cur_indices = np.where(dataset.train_data.ys==lbl)[0]
cur_examples = len(cur_indices)
print('Label, num ex: ', lbl, cur_examples)
cur_op = model.representation
for iex in trange(cur_examples):
cur_im = cur_indices[iex]
x_batch = dataset.train_data.xs[cur_im:cur_im+1,:]
y_batch = dataset.train_data.ys[cur_im:cur_im+1]
dict_nat = {model.x_input: x_batch,
model.y_input: y_batch,
model.is_training: False}
batch_grads = sess.run(cur_op, feed_dict=dict_nat)
if iex==0:
clean_cov = np.zeros(shape=(cur_examples-num_poisoned_left, len(batch_grads)))
full_cov = np.zeros(shape=(cur_examples, len(batch_grads)))
if iex < (cur_examples-num_poisoned_left):
clean_cov[iex]=batch_grads
full_cov[iex] = batch_grads
#np.save(corr_dir+str(lbl)+'_full_cov.npy', full_cov)
total_p = config.data.percentile
clean_mean = np.mean(clean_cov, axis=0, keepdims=True)
full_mean = np.mean(full_cov, axis=0, keepdims=True)
print('Norm of Difference in Mean: ', np.linalg.norm(clean_mean-full_mean))
clean_centered_cov = clean_cov - clean_mean
s_clean = np.linalg.svd(clean_centered_cov, full_matrices=False, compute_uv=False)
print('Top 7 Clean SVs: ', s_clean[0:7])
centered_cov = full_cov - full_mean
u,s,v = np.linalg.svd(centered_cov, full_matrices=False)
print('Top 7 Singular Values: ', s[0:7])
eigs = v[0:1]
p = total_p
corrs = np.matmul(eigs, np.transpose(full_cov)) #shape num_top, num_active_indices
scores = np.linalg.norm(corrs, axis=0) #shape num_active_indices
np.save(os.path.join(model_dir, 'scores.npy'), scores)
print('Length Scores: ', len(scores))
p_score = np.percentile(scores, p)
top_scores = np.where(scores>p_score)[0]
print(top_scores)
num_bad_removed = np.count_nonzero(top_scores>=(len(scores)-num_poisoned_left))
print('Num Bad Removed: ', num_bad_removed)
print('Num Good Rmoved: ', len(top_scores)-num_bad_removed)
num_poisoned_after = num_poisoned_left - num_bad_removed
removed_inds = np.copy(top_scores)
removed_inds_file = os.path.join(model_dir, 'removed_inds.npy')
np.save(removed_inds_file, cur_indices[removed_inds])
print('Num Poisoned Left: ', num_poisoned_after)
if os.path.exists('job_result.json'):
with open('job_result.json') as result_file:
result = json.load(result_file)
result['num_poisoned_left'] = '{}'.format(num_poisoned_after)
else:
result = {'num_poisoned_left': '{}'.format(num_poisoned_after)}
with open('job_result.json', 'w') as result_file:
json.dump(result, result_file, sort_keys=True, indent=4)
def train(config='configs/cifar10_config_stn.json',
save_root_path='/cluster/work/math/fanyang-broglil/CoreRepo',
worstofk=None,
attack_style=None,
attack_limits=None,
lambda_core=None,
num_grouped_ids=None,
num_ids = None,
group_size=None,
use_core=None,
seed=None,
this_repo=None):
config_dict = utilities.get_config(config)
config_dict_copy = copy.deepcopy(config_dict)
# model_dir = config_dict['model']['output_dir']
# if not os.path.exists(model_dir):
# os.makedirs(model_dir)
# # keep the configuration file with the model for reproducibility
# with open(os.path.join(model_dir, 'config.json'), 'w') as f:
# json.dump(config_dict, f, sort_keys=True, indent=4)
config = utilities.config_to_namedtuple(config_dict)
# seeding randomness
if seed == None:
seed = config.training.tf_random_seed
else:
config_dict_copy['training']['tf_random_seed'] = seed
tf.set_random_seed(seed)
np.random.seed(seed)
# Setting up training parameters
max_num_epochs = config.training.max_num_epochs
step_size_schedule = config.training.step_size_schedule
weight_decay = config.training.weight_decay
momentum = config.training.momentum
num_ids = config.training.num_ids # number of IDs per minibatch
if group_size == None:
group_size = config.training.group_size
else:
config_dict_copy['training']['group_size'] = group_size
if num_grouped_ids == None:
num_grouped_ids = config.training.num_grouped_ids
else:
config_dict_copy['training']['num_grouped_ids'] = num_grouped_ids
if num_ids == None:
num_ids = config.training.num_ids
else:
config_dict_copy['training']['num_ids'] = num_ids
if lambda_core == None:
lambda_core = config.training.lambda_
else:
config_dict_copy['training']['lambda_'] = lambda_core
if use_core == None:
use_core = config.training.use_core
else:
config_dict_copy['training']['use_core'] = use_core
adversarial_training = config.training.adversarial_training
eval_during_training = config.training.eval_during_training
if eval_during_training:
num_eval_steps = config.training.num_eval_steps
# Setting up output parameters
num_summary_steps = config.training.num_summary_steps
num_checkpoint_steps = config.training.num_checkpoint_steps
num_easyeval_steps = config.training.num_easyeval_steps
# mini batch size per iteration
# ToDo: need to make this support variable number of num_grouped_ids
batch_size = num_ids + num_grouped_ids
# Setting up model and loss
model_family = config.model.model_family
with_transformer = config.model.transformer
translation_model =config.model.translation_model
if model_family == "resnet":
if with_transformer == True:
if translation_model == "fc":
model = stn_resnet.Model(config.model)
print("Using stn_resnet")
if translation_model == "conv":
model = stn_resnet_conv.Model(config.model)
print("Using stn_resnet_conv")
else:
model = resnet.Model(config.model)
else:
print("Model family does not exist")
exit()
if use_core:
total_loss = model.mean_xent + weight_decay * model.weight_decay_loss + lambda_core * model.core_loss2
else:
total_loss = model.mean_xent + weight_decay * model.weight_decay_loss
# Setting up the data and the model
data_path = config.data.data_path
if config.data.dataset_name == "cifar-10":
raw_cifar = cifar10_input.CIFAR10Data(data_path)
elif config.data.dataset_name == "cifar-100":
raw_cifar = cifar100_input.CIFAR100Data(data_path)
else:
raise ValueError("Unknown dataset name.")
# uncomment to get a list of trainable variables
# model_vars = tf.trainable_variables()
# slim.model_analyzer.analyze_vars(model_vars, print_info=True)
# Setting up the optimizer
boundaries = [int(sss[0]) for sss in step_size_schedule]
boundaries = boundaries[1:]
values = [sss[1] for sss in step_size_schedule]
global_step = tf.train.get_or_create_global_step()
learning_rate = tf.train.piecewise_constant(
tf.cast(global_step, tf.int32),
boundaries,
values)
optimizer = tf.train.MomentumOptimizer(learning_rate, momentum)
#optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate,
# name="Adam")
train_step = optimizer.minimize( total_loss, global_step=global_step)
# Set up adversary
if worstofk == None:
worstofk = config.attack.random_tries
else:
config_dict_copy['attack']['random_tries'] = worstofk
# @ Luzius: incorporate being able to choose multiple transformations
if attack_style == None:
attack_style = 'rotate'
# Training attack
attack = SpatialAttack(model, config.attack, 'random', worstofk, attack_limits)
# Different eval attacks
# Same attack as worstofk
# @ Luzius: currently the names are not clear/consistent since I wasn't sure if we actually want random or not since you originally had your attack like that but I feel like it should rather be worstofk?
# attack_eval_adv = SpatialAttack(model, config.attack, 'random', worstofk, attack_limits)
attack_eval_random = SpatialAttack(model, config.attack, 'random', 1, attack_limits)
# Grid attack
attack_eval_grid = SpatialAttack(model, config.attack, 'grid', None, attack_limits)
# ------------------START EXPERIMENT -------------------------
# Initialize the Repo
print("==> Creating repo..")
# Create repo object if it wasn't passed, comment out if repo has issues
if this_repo == None:
this_repo = exprepo.ExperimentRepo(root_dir=save_root_path)
# Create new experiment
if this_repo != None:
exp_id = this_repo.create_new_experiment('cifar-10',
model_family,
worstofk,
attack_style,
attack_limits,
lambda_core,
num_grouped_ids,
group_size,
config_dict_copy)
# Setting up the Tensorboard and checkpoint outputs
model_dir = '%s/logdir/%s' % (save_root_path, exp_id)
os.makedirs(model_dir, exist_ok=True)
# We add accuracy and xent twice so we can easily make three types of
# comparisons in Tensorboard:
# - train vs eval (for a single run)
# - train of different runs
# - eval of different runs
saver = tf.train.Saver(max_to_keep=3)
tf.summary.scalar('accuracy_nat_train', model.accuracy, collections=['nat'])
tf.summary.scalar('accuracy_nat', model.accuracy, collections = ['nat'])
tf.summary.scalar('xent_nat_train', model.xent / batch_size,
collections=['nat'])
tf.summary.scalar('xent_nat', model.xent / batch_size, collections=['nat'])
tf.summary.image('images_nat_train', model.x_image, collections=['nat'])
tf.summary.scalar('learning_rate', learning_rate, collections=['nat'])
tf.summary.scalar('regression_loss', model.reg_loss, collections=['nat'])
nat_summaries = tf.summary.merge_all('nat')
#dataAugmentation
x_input_placeholder = tf.placeholder(tf.float32,
shape=[None, 32, 32, 3])
flipped = tf.map_fn(lambda img: tf.image.random_flip_left_right(img),
x_input_placeholder)
tot_samp = raw_cifar.train_data.n
max_num_iterations = int(np.floor((tot_samp/num_ids)*max_num_epochs))
print("Total # of samples is: %d; This exp. will run %d iterations" % (tot_samp, max_num_iterations))
# Compute the (epoch) gaps between summary, worstof1eval, checkpoints should happen
summary_gap = int(np.floor(max_num_epochs/num_summary_steps))
easyeval_gap = int(np.floor(max_num_epochs/num_easyeval_steps))
checkpoint_gap = int(np.floor(max_num_epochs/num_checkpoint_steps))
with tf.Session() as sess:
# initialize data augmentation
if config.training.data_augmentation:
if config.data.dataset_name == "cifar-10":
cifar = cifar10_input.AugmentedCIFAR10Data(raw_cifar, sess)
elif config.data.dataset_name == "cifar-100":
cifar = cifar100_input.AugmentedCIFAR100Data(raw_cifar, sess)
else:
raise ValueError("Unknown dataset name.")
else:
cifar = raw_cifar
cifar_eval_dict = {model.x_input: cifar.eval_data.xs,
model.y_input: cifar.eval_data.ys,
model.group: np.arange(0, batch_size, 1, dtype="int32"),
model.transform: np.zeros([cifar.eval_data.n, 3]),
model.is_training: False}
# Initialize the summary writer, global variables, and our time counter.
summary_writer = tf.summary.FileWriter(model_dir, sess.graph)
#if eval_during_training:
eval_dir = os.path.join(model_dir, 'eval')
os.makedirs(eval_dir, exist_ok=True)
eval_summary_writer = tf.summary.FileWriter(eval_dir)
sess.run(tf.global_variables_initializer())
training_time = 0.0
####################################
# Main training loop
####################################
# Initialize cache variables
start_time = time.time()
start_epoch = timer()
it_count = 0
epoch_count = 0
acc_sum = 0
it_summary = 0
it_easyeval = 0
it_ckpt = 0
adv_time = 0
train_time = 0
for ii in range(max_num_iterations+1):
x_batch, y_batch, epoch_done = cifar.train_data.get_next_batch(num_ids, multiple_passes=True)
noop_trans = np.zeros([len(x_batch), 3])
x_batch_nat = x_batch
y_batch_nat = y_batch
id_batch = np.arange(0, num_ids, 1, dtype="int32")
if use_core:
# Create rotated examples
start = timer()
ids = np.arange(0,num_grouped_ids,1,dtype="int32")
for i in range(config.training.group_size):
if config.training.data_augmentation_core:
x_batch_core = sess.run(flipped,feed_dict={x_input_placeholder: x_batch[0:num_grouped_ids,:,:,:]})
else:
x_batch_core = x_batch[0:num_grouped_ids,:,:,:]
x_batch_group, trans_group = attack.perturb(x_batch_core, y_batch[0:num_grouped_ids], sess)
#construct new batches including rotated examples
x_batch_nat = np.concatenate((x_batch_nat, x_batch_group), axis=0)
y_batch_nat = np.concatenate((y_batch_nat, y_batch), axis=0)
noop_trans = np.concatenate((noop_trans, trans_group), axis=0)
id_batch = np.concatenate((id_batch, ids), axis=0)
end = timer()
training_time += end - start
adv_time += end - start
else:
if adversarial_training:
start = timer()
x_batch_nat, noop_trans = attack.perturb(x_batch, y_batch, sess)
end = timer()
adv_time += end - start
else:
x_batch_nat, noop_trans = x_batch, noop_trans
nat_dict = {model.x_input: x_batch_nat,
model.y_input: y_batch_nat,
model.group: id_batch,
model.transform: noop_trans,
model.is_training: False}
################# Outputting/saving weights and evaluations ###############
nat_acc = -1.0
acc_grid = -1.0
avg_xent_grid = -1.0
saved_weights = 0
# Compute training accuracy on this minibatch
train_nat_acc = sess.run(model.accuracy, feed_dict=nat_dict)
# Output to stdout
if epoch_done:
epoch_time = timer() - start_epoch
# Average
av_acc = acc_sum/it_count
# ToDo: Log this to file as well
# Training accuracy over epoch
print('Epoch {}: ({})'.format(epoch_count, datetime.now()))
print(' training natural accuracy {:.4}%'.format(av_acc * 100))
print(' {:.4} seconds per epoch'.format(epoch_time))
# Accuracy on entire test set
test_nat_acc = sess.run(model.accuracy, feed_dict=cifar_eval_dict)
print(' test set natural accuracy {:.4}%'.format(test_nat_acc * 100))
# print(' {:.4} seconds for test evaluation'.format(test_time))
print("example TIME")
print(adv_time)
print("train TIME")
print(train_time)
########### Things to do every xxx epochs #############
# Check if worstof1 eval should be run
if it_summary == summary_gap - 1 or epoch_count == max_num_epochs - 1:
summary = sess.run(nat_summaries, feed_dict=nat_dict)
summary_writer.add_summary(summary, global_step.eval(sess))
it_summary = 0
else:
it_summary += 1
if it_easyeval == easyeval_gap - 1 or epoch_count == max_num_epochs - 1:
# Evaluation on adv and natural
[acc_nat, acc_adv, avg_xent_nat, avg_xent_adv] = evaluate(model, attack_eval_random, sess, config, "random", data_path, None)
# Save in checkpoint
chkpt_id = this_repo.create_training_checkpoint(
exp_id, training_step=ii,
epoch=epoch_count,
train_acc_nat=nat_acc,
test_acc_adv=acc_adv, test_acc_nat=acc_nat,
test_loss_adv=avg_xent_adv,
test_loss_nat=avg_xent_nat)
it_easyeval = 0
else:
it_easyeval += 1
startt = timer()
if it_ckpt == checkpoint_gap - 1 or epoch_count == max_num_epochs - 1:
# Create checkpoint id if non-existent
if not chkpt_id :
chkpt_id = this_repo.create_training_checkpoint(
exp_id, training_step=ii,
epoch=epoch_count,
train_acc_nat=train_nat_acc,
test_acc_nat=test_nat_acc)
# Save checkpoint data (weights)
saver.save(sess,
os.path.join(model_dir, '{}_checkpoint'.format(chkpt_id)))
print(' chkpt saving took {:.4}s '.format(timer()-startt))
it_ckpt = 0
else:
it_ckpt += 1
# Set loss sum, it count back to zero
acc_sum = train_nat_acc
epoch_done = 0
epoch_count += 1
start_epoch = timer()
it_count = 1
else:
it_count += 1
acc_sum += train_nat_acc
# Actual training step
start = timer()
nat_dict[model.is_training] = True
sess.run(train_step, feed_dict=nat_dict)
training_time += timer() - start
train_time += timer() - start
runtime = time.time() - start_time
# Do all evaluations in the last step - on grid
[_, acc_grid, _, avg_xent_grid] = evaluate(model, attack_eval_grid, sess, config, "grid", data_path, eval_summary_writer)
this_repo.mark_experiment_as_completed(
exp_id, train_acc_nat=nat_acc,
test_acc_adv=acc_adv, test_acc_nat=acc_nat,
test_acc_grid=acc_grid, runtime=runtime)
return 0
