def PDP_resnet_with_pretrain_adv(TIN_data, resnet_params, train_params, params_to_save):
# dict for encoding layer variables and output layer variables
pre_define_vars = {}
# list of variables to train
train_vars = []
pretrain_vars = []
with tf.Graph().as_default(), tf.device('/cpu:0'):
global_step = tf.Variable(0, trainable=False)
# Parameters Declarification
######################################
# encoding (pretrain) layer variables
with tf.variable_scope('enc_layer', reuse=tf.AUTO_REUSE) as scope:
kernel1 = tf.get_variable('kernel1', shape=[train_params.enc_kernel_size, train_params.enc_kernel_size,
3, train_params.enc_filters], dtype=tf.float32,
initializer=tf.contrib.layers.xavier_initializer_conv2d())
biases1 = tf.get_variable('biases1', shape=[train_params.enc_filters], dtype=tf.float32,
initializer=tf.constant_initializer(0.0))
pre_define_vars['kernel1'] = kernel1
pre_define_vars['biases1'] = biases1
train_vars.append(kernel1)
train_vars.append(biases1)
pretrain_vars.append(kernel1)
pretrain_vars.append(biases1)
shape = kernel1.get_shape().as_list()
w_t = tf.reshape(kernel1, [-1, shape[-1]])
w = tf.transpose(w_t)
sing_vals = tf.svd(w, compute_uv=False)
sensitivity = tf.reduce_max(sing_vals)
gamma = 2*train_params.Delta2/(train_params.effective_batch_size * sensitivity)
print('gamma: {}'.format(gamma))
# output layer variables
with tf.variable_scope('fc2', reuse=tf.AUTO_REUSE) as scope:
stdv = 1.0 / math.sqrt(train_params.hk)
final_w = tf.get_variable('kernel', shape=[train_params.hk, train_params.num_classes], dtype=tf.float32,
initializer=tf.random_uniform_initializer(-stdv, stdv))
final_b = tf.get_variable('bias', shape=[train_params.num_classes], dtype=tf.float32,
initializer=tf.constant_initializer(0.0))
pre_define_vars['final_w'] = final_w
pre_define_vars['final_b'] = final_b
train_vars.append(final_w)
train_vars.append(final_b)
######################################
# Build a Graph that computes the logits predictions from the inputs
######################################
# input placeholders
x_sb = tf.placeholder(tf.float32, [None,train_params.image_size,train_params.image_size,3], name='x_sb') # input is the bunch of n_batchs
x_sb_adv = tf.placeholder(tf.float32, [None,train_params.image_size,train_params.image_size,3], name='x_sb_adv')
x_test = tf.placeholder(tf.float32, [None,train_params.image_size,train_params.image_size,3], name='x_test')
y_sb = tf.placeholder(tf.float32, [None, train_params.num_classes], name='y_sb') # input is the bunch of n_batchs (super batch)
y_sb_adv = tf.placeholder(tf.float32, [None, train_params.num_classes], name='y_sb_adv')
y_test = tf.placeholder(tf.float32, [None, train_params.num_classes], name='y_test')
FM_h = tf.placeholder(tf.float32, [None, train_params.enc_h_size, train_params.enc_h_size, train_params.enc_filters], name='FM_h') # one time
noise = tf.placeholder(tf.float32, [None, train_params.image_size, train_params.image_size, 3], name='noise') # one time
adv_noise = tf.placeholder(tf.float32, [None, train_params.image_size, train_params.image_size, 3], name='adv_noise') # one time
learning_rate = tf.placeholder(tf.float32, shape=(), name='learning_rate')
keep_prob = tf.placeholder(tf.float32, shape=(), name='keep_prob')
# list of grads for each GPU
tower_pretrain_grads = []
tower_train_grads = []
all_train_loss = []
# optimizers
pretrain_opt = tf.train.AdamOptimizer(learning_rate)
train_opt = tf.train.AdamOptimizer(learning_rate)
# model and loss on one GPU
with tf.device('/gpu:{}'.format(GPU_IDX[0])):
# setup encoding layer training
with tf.variable_scope('enc_layer', reuse=tf.AUTO_REUSE) as scope:
Enc_Layer2 = EncLayer(inpt=x_sb, n_filter_in=None, n_filter_out=None, filter_size=None,
W=kernel1, b=biases1, activation=tf.nn.relu)
pretrain_adv = Enc_Layer2.get_train_ops2(xShape=tf.shape(x_sb_adv)[0], Delta=train_params.Delta2,
epsilon=train_params.epsilon2, batch_size=None, learning_rate=None,
W=kernel1, b=biases1, perturbFMx=adv_noise, perturbFM_h=FM_h)
Enc_Layer3 = EncLayer(inpt=x_sb, n_filter_in=None, n_filter_out=None, filter_size=None,
W=kernel1, b=biases1, activation=tf.nn.relu)
pretrain_benign = Enc_Layer3.get_train_ops2(xShape=tf.shape(x_sb)[0], Delta=train_params.Delta2,
epsilon=train_params.epsilon2, batch_size=None, learning_rate=None,
W=kernel1, b=biases1, perturbFMx=noise, perturbFM_h=FM_h)
pretrain_cost = tf.reduce_mean(pretrain_adv + pretrain_benign)
print_var('pretrain_cost', pretrain_cost)
# use standard loss first
y_logits = inference(x_sb + noise, FM_h, keep_prob, pre_define_vars, resnet_params, train_params)
y_softmax = tf.nn.softmax(y_logits)
y_logits_adv = inference(x_sb_adv + adv_noise, FM_h, keep_prob, pre_define_vars, resnet_params, train_params)
y_softmax_adv = tf.nn.softmax(y_logits_adv)
# taylor exp
# TODO: use noise here
perturbW = train_params.perturbFM * final_w
# train_loss = TaylorExp_no_noise(y_softmax, y_sb, y_softmax_adv, y_sb_adv,
# train_params.effective_batch_size, train_params.alpha)
train_loss = TaylorExp(y_softmax, y_sb, y_softmax_adv, y_sb_adv,
train_params.effective_batch_size, train_params.alpha, perturbW)
print_var('train_loss', train_loss)
all_train_loss.append(train_loss)
# split testing in each gpu
x_sb_tests = tf.split(x_sb, N_ALL_GPUS, axis=0)
y_softmax_test_list = []
for gpu in range(N_ALL_GPUS):
with tf.device('/gpu:{}'.format(gpu)):
# testing graph now in each gpu
y_logits_test = test_inference(x_sb_tests[gpu] + noise, FM_h, keep_prob, pre_define_vars, resnet_params, train_params)
y_softmax_test_list.append(tf.nn.softmax(y_logits_test))
y_softmax_test_concat = tf.concat(y_softmax_test_list, axis=0)
print('$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$')
all_vars = tf.global_variables()
print_var_list('all vars', all_vars)
print('$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$')
# add selected vars into trainable variable list
# ('res4' in var.name and ('gamma' in var.name or 'beta' in var.name)) or
for var in tf.global_variables():
if 'resnet_model' in var.name and \
('conv0' in var.name or
'fc' in var.name or
'res3' in var.name or
'res4' in var.name or
'res1' in var.name or
'res2' in var.name) and \
('gamma' in var.name or
'beta' in var.name or
'kernel' in var.name or
'bias' in var.name):
if var not in train_vars:
train_vars.append(var)
elif 'enc_layer' in var.name and \
('kernel' in var.name or
'bias' in var.name):
if var not in pretrain_vars:
pretrain_vars.append(var)
if var not in train_vars:
train_vars.append(var)
elif 'enc_layer' in var.name and \
('gamma' in var.name or
'beta' in var.name):
if var not in pretrain_vars:
pretrain_vars.append(var)
print('$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$')
print_var_list('train_vars', train_vars)
print_var_list('pretrain_vars', pretrain_vars)
print('$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$')
# op for compute grads on one gpu
with tf.device('/gpu:{}'.format(GPU_IDX[0])):
# get all update_ops (updates of moving averageand std) for batch normalizations
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
print_op_list('update ops', update_ops)
enc_update_ops = [op for op in update_ops if 'enc_layer' in op.name]
print_op_list('enc layer update ops', enc_update_ops)
# when the gradients are computed, update the batch_norm
with tf.control_dependencies(enc_update_ops):
pretrain_grads = pretrain_opt.compute_gradients(pretrain_cost, var_list=pretrain_vars)
print('*********** pretrain_grads ***********')
for x in pretrain_grads:
print(x)
print('**********************')
with tf.control_dependencies(update_ops):
train_grads = train_opt.compute_gradients(train_loss, var_list=train_vars)
print('*********** train_grads ***********')
for x in train_grads:
print(x)
print('**********************')
avg_pretrain_grads = pretrain_grads
avg_train_grads = train_grads
# get averaged loss tensor for pretrain and train ops
total_loss = tf.reduce_sum(tf.stack(all_train_loss))
total_pretrain_loss = tf.reduce_mean(pretrain_cost)
# prepare to save gradients for large batch
pretrain_grads_save = [g for g,v in pretrain_grads]
# print('*********** pretrain_grads_save ***********' + str(pretrain_grads_save) + '**********************')
train_grads_save = [g for g,v in train_grads]
# print('*********** train_grads_save ***********' + str(train_grads_save) + '**********************')
pretrain_grads_shapes = [g.shape.as_list() for g in pretrain_grads_save]
train_grads_shapes = [g.shape.as_list() for g in train_grads_save]
# placeholders for importing saved gradients
pretrain_grads_placeholders = []
for g,v in pretrain_grads:
pretrain_grads_placeholders.append(tf.placeholder(tf.float32, v.shape))
train_grads_placeholders = []
for g,v in train_grads:
train_grads_placeholders.append(tf.placeholder(tf.float32, v.shape))
# construct the (grad, var) list
assemble_pretrain_grads = []
for i in range(len(pretrain_vars)):
assemble_pretrain_grads.append((pretrain_grads_placeholders[i], pretrain_vars[i]))
assemble_train_grads = []
for i in range(len(train_grads)):
assemble_train_grads.append((train_grads_placeholders[i], train_vars[i]))
# apply the saved gradients
pretrain_op = pretrain_opt.apply_gradients(assemble_pretrain_grads, global_step=global_step)
train_op = train_opt.apply_gradients(assemble_train_grads, global_step=global_step)
######################################
# Create a saver.
saver = tf.train.Saver(var_list=tf.all_variables(), max_to_keep=1000)
# start a session with memory growth
config = tf.ConfigProto(log_device_placement=False)
config.gpu_options.allow_growth=True
sess = tf.Session(config=config)
print("session created")
# get some initial values
sess.run(kernel1.initializer)
_gamma = sess.run(gamma)
_gamma_x = train_params.Delta2 / train_params.effective_batch_size
epsilon2_update = train_params.epsilon2/(1.0 + 1.0/_gamma + 1/_gamma_x)
delta_r = train_params.fgsm_eps * (train_params.image_size ** 2)
_sensitivityW = sess.run(sensitivity)
delta_h = _sensitivityW*(train_params.enc_h_size ** 2)
#dp_mult = (train_params.Delta2 / (train_params.effective_batch_size * epsilon2_update)) / (delta_r / train_params.dp_epsilon) + \
# (2 * train_params.Delta2 / (train_params.effective_batch_size * epsilon2_update))/(delta_h / train_params.dp_epsilon)
dp_mult = (train_params.Delta2*train_params.dp_epsilon) / (train_params.effective_batch_size*epsilon2_update * (delta_h / 2 + delta_r))
# save some valus for testing
params_to_save['epsilon2_update'] = epsilon2_update
params_to_save['dp_mult'] = dp_mult
#######################################
# ADV attacks
#######################################
# split input for attacks
x_attacks = tf.split(x_sb, 3, axis=0) # split it into each batch
# currently only ifgsm, mim, and madry attacks are available
attack_switch = {'fgsm':False, 'ifgsm':True, 'deepfool':False, 'mim':True, 'spsa':False, 'cwl2':False, 'madry':True, 'stm':False}
# wrap the inference
ch_model_probs = CustomCallableModelWrapper(callable_fn=inference_test_output_probs, output_layer='probs',
adv_noise=adv_noise, keep_prob=keep_prob, pre_define_vars=pre_define_vars,
resnet_params=resnet_params, train_params=train_params)
# to save the reference to the attack tensors
attack_tensor_training_dict = {}
attack_tensor_testing_dict = {}
# placeholder for eps parameter
mu_alpha = tf.placeholder(tf.float32, [1])
# Iterative FGSM (BasicIterativeMethod/ProjectedGradientMethod with no random init)
# place on specific GPU
with tf.device('/gpu:{}'.format(AUX_GPU_IDX[0])):
print('ifgsm GPU placement')
print('/gpu:{}'.format(AUX_GPU_IDX[0]))
if attack_switch['ifgsm']:
print('creating attack tensor of BasicIterativeMethod')
ifgsm_obj = BasicIterativeMethod(model=ch_model_probs, sess=sess)
attack_tensor_training_dict['ifgsm'] = ifgsm_obj.generate(x=x_attacks[0], eps=mu_alpha, eps_iter=mu_alpha/train_params.iter_step_training, nb_iter=train_params.iter_step_training, clip_min=-1.0, clip_max=1.0)
attack_tensor_testing_dict['ifgsm'] = ifgsm_obj.generate(x=x_sb, eps=mu_alpha, eps_iter=mu_alpha/train_params.iter_step_testing, nb_iter=train_params.iter_step_testing, clip_min=-1.0, clip_max=1.0)
# MomentumIterativeMethod
# place on specific GPU
with tf.device('/gpu:{}'.format(AUX_GPU_IDX[1])):
print('mim GPU placement')
print('/gpu:{}'.format(AUX_GPU_IDX[1]))
if attack_switch['mim']:
print('creating attack tensor of MomentumIterativeMethod')
mim_obj = MomentumIterativeMethod(model=ch_model_probs, sess=sess)
attack_tensor_training_dict['mim'] = mim_obj.generate(x=x_attacks[1], eps=mu_alpha, eps_iter=mu_alpha/train_params.iter_step_training, nb_iter=train_params.iter_step_training, decay_factor=1.0, clip_min=-1.0, clip_max=1.0)
attack_tensor_testing_dict['mim'] = mim_obj.generate(x=x_sb, eps=mu_alpha, eps_iter=mu_alpha/train_params.iter_step_testing, nb_iter=train_params.iter_step_testing, decay_factor=1.0, clip_min=-1.0, clip_max=1.0)
# MadryEtAl (Projected Grdient with random init, same as rand+fgsm)
# place on specific GPU
with tf.device('/gpu:{}'.format(AUX_GPU_IDX[2])):
print('madry GPU placement')
print('/gpu:{}'.format(AUX_GPU_IDX[2]))
if attack_switch['madry']:
print('creating attack tensor of MadryEtAl')
madry_obj = MadryEtAl(model=ch_model_probs, sess=sess)
attack_tensor_training_dict['madry'] = madry_obj.generate(x=x_attacks[2], eps=mu_alpha, eps_iter=mu_alpha/train_params.iter_step_training, nb_iter=train_params.iter_step_training, clip_min=-1.0, clip_max=1.0)
attack_tensor_testing_dict['madry'] = madry_obj.generate(x=x_sb, eps=mu_alpha, eps_iter=mu_alpha/train_params.iter_step_testing, nb_iter=train_params.iter_step_testing, clip_min=-1.0, clip_max=1.0)
# combine the tensors
adv_tensors_concat = tf.concat([attack_tensor_training_dict[x] for x in train_params.attacks], axis=0)
#######################################
# init op
print('initialize_all_variables')
init = tf.initialize_all_variables()
sess.run(init)
# load pretrained variables of RESNET
if train_params.load_weights:
# first we need to load variable name convert table
tgt_var_name_dict = {}
with open(train_params.weight_table_path, 'r', encoding='utf-8') as inf:
lines = inf.readlines()
for line in lines:
var_names = line.strip().split(' ')
if var_names[1] == 'NONE':
continue
else:
tgt_var_name_dict[var_names[0]] = var_names[1]
# load variables dict from checkpoint
pretrained_var_dict = load_pretrained_vars()
# load pre-trained vars using name convert table
for var in tf.global_variables():
if var.name in tgt_var_name_dict:
# print('var \"{}\" found'.format(var.name))
try:
var.load(pretrained_var_dict[tgt_var_name_dict[var.name]], session=sess)
print('{} loaded'.format(var.name))
except:
print('var {} not loaded since shape changed'.format(var.name))
else:
if 'Adam' not in var.name:
print('var \"{}\" NOT FOUND'.format(var.name))
else:
print('Training model from scratch')
#####################################
# init noise and save for testing
perturbH_test = np.random.laplace(0.0, 0, train_params.enc_h_size*train_params.enc_h_size*train_params.enc_filters)
perturbH_test = np.reshape(perturbH_test, [-1, train_params.enc_h_size, train_params.enc_h_size, train_params.enc_filters])
params_to_save['perturbH_test'] = perturbH_test
perturbFM_h = np.random.laplace(0.0, 2*train_params.Delta2/(epsilon2_update*train_params.effective_batch_size),
train_params.enc_h_size*train_params.enc_h_size*train_params.enc_filters)
perturbFM_h = np.reshape(perturbFM_h, [-1, train_params.enc_h_size, train_params.enc_h_size, train_params.enc_filters])
params_to_save['perturbFM_h'] = perturbFM_h
Noise = generateIdLMNoise(train_params.image_size, train_params.Delta2, epsilon2_update, train_params.effective_batch_size)
params_to_save['Noise'] = Noise
Noise_test = generateIdLMNoise(train_params.image_size, 0, epsilon2_update, train_params.effective_batch_size)
params_to_save['Noise_test'] = Noise_test
# save params for testing
with open(os.getcwd() + train_params.params_save_path, 'wb') as outf:
pickle.dump(params_to_save, outf)
print('params saved')
####################################
print('start pretrain')
start_time = time.time()
lr_schedule_list = sorted(train_params.lr_schedule_pretrain.keys())
attacks_and_benign = train_params.attacks + ['benign']
# build zeros numpy arrays for accumulate grads
accumu_pretrain_grads = [np.zeros(g_shape, dtype=np.float32) for g_shape in pretrain_grads_shapes]
total_pretrain_loss_value = 0.0
step = 0
# pretrain loop
while True:
# if enough steps, break
if step > train_params.pretrain_steps:
break
# add steps here so not forgot
else:
step += 1
# manual schedule learning rate
current_epoch = step // (train_params.epoch_steps)
current_lr = train_params.lr_schedule_pretrain[get_lr(current_epoch, lr_schedule_list)]
# benign and adv batch
super_batch = TIN_data.train.next_super_batch(N_GPUS, ensemble=False, random=True)
adv_super_batch = TIN_data.train.next_super_batch(N_GPUS, ensemble=False, random=True)
# get pretrain grads
pretrain_grads_save_np, _pretain_loss_value = sess.run([pretrain_grads_save, total_pretrain_loss], feed_dict={x_sb: super_batch[0],
x_sb_adv: adv_super_batch[0],
learning_rate: current_lr,
adv_noise: Noise_test,
noise: Noise,
FM_h: perturbFM_h})
# accumulate grads
for i in range(len(accumu_pretrain_grads)):
accumu_pretrain_grads[i] = accumu_pretrain_grads[i] + pretrain_grads_save_np[i]
# accumulate loss values
total_pretrain_loss_value = total_pretrain_loss_value + _pretain_loss_value
# use accumulated gradients to update variables
if step % train_params.batch_multi == 0 and step > 0:
# print('effective batch reached at step: {}, epoch: {}'.format(step, step / train_params.epoch_steps))
# compute the average grads and build the feed dict
pretrain_feed_dict = {}
for i in range(len(accumu_pretrain_grads)):
pretrain_feed_dict[pretrain_grads_placeholders[i]] = accumu_pretrain_grads[i] / train_params.batch_multi
pretrain_feed_dict[learning_rate] = current_lr
# run train ops by feeding the gradients
sess.run(pretrain_op, feed_dict=pretrain_feed_dict)
# get loss value
avg_pretrain_loss_value = total_pretrain_loss_value / train_params.batch_multi
# reset the average grads
accumu_pretrain_grads = [np.zeros(g_shape, dtype=np.float32) for g_shape in pretrain_grads_shapes]
total_pretrain_loss_value = 0.0
# print loss
if step % (1*train_params.epoch_steps) == 0 and step >= (1*train_params.epoch_steps):
print('pretrain report at step: {}, epoch: {}'.format(step, step / train_params.epoch_steps))
dt = time.time() - start_time
avg_epoch_time = dt / (step / train_params.epoch_steps)
print('epoch: {:.4f}, avg epoch time: {:.4f}, current_lr: {}'.format(step/train_params.epoch_steps, avg_epoch_time, current_lr), flush=True)
print('pretrain_loss: {:.6f}'.format(avg_pretrain_loss_value))
####################################
print('start train')
start_time = time.time()
lr_schedule_list = sorted(train_params.lr_schedule.keys())
# train whole model
# build zeros numpy arrays for accumulate grads
accumu_pretrain_grads = [np.zeros(g_shape, dtype=np.float32) for g_shape in pretrain_grads_shapes]
accumu_train_grads = [np.zeros(g_shape, dtype=np.float32) for g_shape in train_grads_shapes]
total_pretrain_loss_value = 0.0
total_train_loss_value = 0.0
step = 0
# train loop
while True:
# if enough steps, break
if step > train_params.train_steps:
break
# add steps here so not forgot
else:
step += 1
# compute the grads every step
# random eps value for trianing
d_eps = random.random()*train_params.random_eps_range
# manual schedule learning rate
current_epoch = step // (train_params.epoch_steps)
current_lr = train_params.lr_schedule[get_lr(current_epoch, lr_schedule_list)]
# benign and adv batch
super_batch = TIN_data.train.next_super_batch(N_GPUS, ensemble=False, random=True)
adv_super_batch = TIN_data.train.next_super_batch(N_GPUS, ensemble=False, random=True)
# create adv samples
super_batch_adv_images = sess.run(adv_tensors_concat,
feed_dict={x_sb:adv_super_batch[0], keep_prob:1.0,
adv_noise: Noise, mu_alpha:[d_eps]})
# get pretrain and train grads
pretrain_grads_save_np, _pretain_loss_value = sess.run([pretrain_grads_save, total_pretrain_loss], feed_dict={x_sb: super_batch[0],
x_sb_adv: super_batch_adv_images,
learning_rate: current_lr,
adv_noise: Noise_test,
noise: Noise,
FM_h: perturbFM_h})
train_grads_save_np, _train_loss_value = sess.run([train_grads_save, total_loss], feed_dict = {x_sb: super_batch[0], y_sb: super_batch[1],
x_sb_adv: super_batch_adv_images, y_sb_adv: adv_super_batch[1],
keep_prob: train_params.keep_prob, learning_rate: current_lr,
noise: Noise, adv_noise: Noise_test, FM_h: perturbFM_h})
# accumulate grads
for i in range(len(accumu_pretrain_grads)):
accumu_pretrain_grads[i] = accumu_pretrain_grads[i] + pretrain_grads_save_np[i]
for i in range(len(accumu_train_grads)):
accumu_train_grads[i] = accumu_train_grads[i] + train_grads_save_np[i]
# accumulate loss values
total_pretrain_loss_value = total_pretrain_loss_value + _pretain_loss_value
total_train_loss_value = total_train_loss_value + _train_loss_value
# use accumulated gradients to update variables
if step % train_params.batch_multi == 0 and step > 0:
# compute the average grads and build the feed dict
pretrain_feed_dict = {}
for i in range(len(accumu_pretrain_grads)):
pretrain_feed_dict[pretrain_grads_placeholders[i]] = accumu_pretrain_grads[i] / train_params.batch_multi
pretrain_feed_dict[learning_rate] = current_lr
# pretrain_feed_dict[keep_prob] = 0.5
train_feed_dict = {}
for i in range(len(accumu_train_grads)):
train_feed_dict[train_grads_placeholders[i]] = accumu_train_grads[i] / train_params.batch_multi
train_feed_dict[learning_rate] = current_lr
# train_feed_dict[keep_prob] = 0.5
# run train ops
sess.run(pretrain_op, feed_dict=pretrain_feed_dict)
sess.run(train_op, feed_dict=train_feed_dict)
# get loss value
avg_pretrain_loss_value = total_pretrain_loss_value / train_params.batch_multi
avg_train_loss_value = total_train_loss_value / train_params.batch_multi
# reset the average grads
accumu_pretrain_grads = [np.zeros(g_shape, dtype=np.float32) for g_shape in pretrain_grads_shapes]
accumu_train_grads = [np.zeros(g_shape, dtype=np.float32) for g_shape in train_grads_shapes]
total_pretrain_loss_value = 0.0
total_train_loss_value = 0.0
# print status every epoch
if step % int(train_params.epoch_steps) == 0:
dt = time.time() - start_time
avg_epoch_time = dt / (step / train_params.epoch_steps)
print('epoch: {:.4f}, avg epoch time: {:.4f}s, current_lr: {}'.format(step/train_params.epoch_steps, avg_epoch_time, current_lr), flush=True)
# save model
if step % int(train_params.epoch_steps) == 0 and int(step / train_params.epoch_steps) in train_params.epochs_to_save:
print('saving model at epoch {}'.format(step / train_params.epoch_steps))
checkpoint_path = os.path.join(os.getcwd() + train_params.check_point_dir, 'model.ckpt')
saver.save(sess, checkpoint_path, global_step=step)
# testing during training
if step % int(train_params.epoch_steps) == 0 and int(step / train_params.epoch_steps) in train_params.epochs_to_test:
test_start = time.time()
print('train test reported at step: {}, epoch: {}'.format(step, step / train_params.epoch_steps))
dt = time.time() - start_time
avg_epoch_time = dt / (step / train_params.epoch_steps)
print('epoch: {:.4f}, avg epoch time: {:.4f}s, current_lr: {}'.format(step/train_params.epoch_steps, avg_epoch_time, current_lr), flush=True)
print('pretrain_loss: {:.6f}, train_loss: {:.6f}'.format(avg_pretrain_loss_value, avg_train_loss_value))
# print('output layer: \n\t{}'.format(output_layer_value))
#===================adv samples=====================
adv_acc_dict = {}
robust_adv_acc_dict = {}
robust_adv_utility_dict = {}
log_str = ''
# cover all test data
for i in range(train_params.test_epochs):
test_batch = TIN_data.test.next_batch(train_params.test_batch_size)
# if more GPUs available, generate testing adv samples at once
if N_AUX_GPUS > 1:
adv_images_dict = sess.run(attack_tensor_testing_dict, feed_dict ={x_sb: test_batch[0],
adv_noise: Noise_test,
mu_alpha: [train_params.fgsm_eps],
keep_prob: 1.0})
else:
adv_images_dict = {}
# test for each attack
for atk in attacks_and_benign:
if atk not in adv_acc_dict:
adv_acc_dict[atk] = 0.0
robust_adv_acc_dict[atk] = 0.0
robust_adv_utility_dict[atk] = 0.0
if atk == 'benign':
testing_img = test_batch[0]
elif attack_switch[atk]:
# if only one gpu available, generate adv samples in-place
if atk not in adv_images_dict:
adv_images_dict[atk] = sess.run(attack_tensor_testing_dict[atk], feed_dict ={x_sb:test_batch[0],
adv_noise: Noise_test,
mu_alpha:[train_params.fgsm_eps],
keep_prob: 1.0})
testing_img = adv_images_dict[atk]
else:
continue
### PixelDP Robustness ###
predictions_form_argmax = np.zeros([train_params.test_batch_size, train_params.num_classes])
softmax_predictions = sess.run(y_softmax_test_concat, feed_dict={x_sb: testing_img, noise: Noise, FM_h: perturbFM_h, keep_prob: 1.0})
argmax_predictions = np.argmax(softmax_predictions, axis=1)
for n_draws in range(0, train_params.num_samples):
_BenignLNoise = generateIdLMNoise(train_params.image_size, train_params.Delta2, epsilon2_update, train_params.effective_batch_size)
_perturbFM_h = np.random.laplace(0.0, 2*train_params.Delta2/(epsilon2_update*train_params.effective_batch_size),
train_params.enc_h_size*train_params.enc_h_size*train_params.enc_filters)
_perturbFM_h = np.reshape(_perturbFM_h, [-1, train_params.enc_h_size, train_params.enc_h_size, train_params.enc_filters])
for j in range(train_params.test_batch_size):
pred = argmax_predictions[j]
predictions_form_argmax[j, pred] += 1
softmax_predictions = sess.run(y_softmax_test_concat, feed_dict={x_sb: testing_img, noise: (_BenignLNoise/10 + Noise), FM_h: perturbFM_h, keep_prob: 1.0}) * \
sess.run(y_softmax_test_concat, feed_dict={x_sb: testing_img, noise: Noise, FM_h: (_perturbFM_h/10 + perturbFM_h), keep_prob: 1.0})
argmax_predictions = np.argmax(softmax_predictions, axis=1)
final_predictions = predictions_form_argmax
is_correct = []
is_robust = []
for j in range(train_params.test_batch_size):
is_correct.append(np.argmax(test_batch[1][j]) == np.argmax(final_predictions[j]))
robustness_from_argmax = robustness.robustness_size_argmax(counts=predictions_form_argmax[j],
eta=0.05, dp_attack_size=train_params.fgsm_eps,
dp_epsilon=train_params.dp_epsilon, dp_delta=0.05,
dp_mechanism='laplace') / dp_mult
is_robust.append(robustness_from_argmax >= train_params.fgsm_eps)
adv_acc_dict[atk] += np.sum(is_correct)*1.0/train_params.test_batch_size
robust_adv_acc_dict[atk] += np.sum([a and b for a,b in zip(is_robust, is_correct)])*1.0/np.sum(is_robust)
robust_adv_utility_dict[atk] += np.sum(is_robust)*1.0/train_params.test_batch_size
##############################
# average all acc for whole test data
for atk in attacks_and_benign:
adv_acc_dict[atk] = adv_acc_dict[atk] / train_params.test_epochs
robust_adv_acc_dict[atk] = robust_adv_acc_dict[atk] / train_params.test_epochs
robust_adv_utility_dict[atk] = robust_adv_utility_dict[atk] / train_params.test_epochs
# added robust prediction
log_str += " {}: {:.6f} {:.6f} {:.6f} {:.6f}\n".format(atk, adv_acc_dict[atk], robust_adv_acc_dict[atk], robust_adv_utility_dict[atk], robust_adv_acc_dict[atk] * robust_adv_utility_dict[atk])
dt = time.time() - test_start
print('testing time: {}'.format(dt))
print(log_str, flush=True)
print('*******************')
def SSGD_resnet_testing(TIN_data, resnet_params, train_params, test_params,
all_params):
# dict for encoding layer variables and output layer variables
pre_define_vars = {}
# list of variables to train
train_vars = []
with tf.Graph().as_default(), tf.device('/cpu:0'):
global_step = tf.Variable(0, trainable=False)
# Parameters Declarification
######################################
# encoding (pretrain) layer variables
with tf.variable_scope('enc_layer', reuse=tf.AUTO_REUSE) as scope:
kernel1 = tf.get_variable(
'kernel1',
shape=[
train_params.enc_kernel_size, train_params.enc_kernel_size,
3, train_params.enc_filters
],
dtype=tf.float32,
initializer=tf.contrib.layers.xavier_initializer_conv2d())
biases1 = tf.get_variable('biases1',
shape=[train_params.enc_filters],
dtype=tf.float32,
initializer=tf.constant_initializer(0.0))
pre_define_vars['kernel1'] = kernel1
pre_define_vars['biases1'] = biases1
train_vars.append(kernel1)
train_vars.append(biases1)
dp_mult = all_params['dp_mult']
# output layer variables
with tf.variable_scope('fc2', reuse=tf.AUTO_REUSE) as scope:
stdv = 1.0 / math.sqrt(train_params.hk)
final_w = tf.get_variable(
'kernel',
shape=[train_params.hk, train_params.num_classes],
dtype=tf.float32,
initializer=tf.random_uniform_initializer(-stdv, stdv))
final_b = tf.get_variable('bias',
shape=[train_params.num_classes],
dtype=tf.float32,
initializer=tf.constant_initializer(0.0))
pre_define_vars['final_w'] = final_w
pre_define_vars['final_b'] = final_b
train_vars.append(final_w)
train_vars.append(final_b)
######################################
# Build a Graph that computes the logits predictions from the inputs
######################################
# input placeholders
x_sb = tf.placeholder(
tf.float32,
[None, train_params.image_size, train_params.image_size, 3],
name='x_sb') # input is the bunch of n_batchs
x_test = tf.placeholder(
tf.float32,
[None, train_params.image_size, train_params.image_size, 3],
name='x_test')
y_sb = tf.placeholder(
tf.float32, [None, train_params.num_classes],
name='y_sb') # input is the bunch of n_batchs (super batch)
y_test = tf.placeholder(tf.float32, [None, train_params.num_classes],
name='y_test')
noise = tf.placeholder(tf.float32, [
None, train_params.enc_h_size, train_params.enc_h_size,
train_params.enc_filters
],
name='noise') # one time
keep_prob = tf.placeholder(tf.float32, shape=(), name='keep_prob')
with tf.device('/gpu:0'):
# the model for testing
y_logits_test, _ = test_inference(
x_sb, train_params.attack_norm_bound * noise, keep_prob,
pre_define_vars, resnet_params, train_params)
y_softmax_test = tf.nn.softmax(y_logits_test)
correct_prediction = tf.equal(tf.argmax(y_logits_test, 1),
tf.argmax(y_sb, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
# print all variables
print('$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$')
all_vars = tf.global_variables()
print_var_list('all vars', all_vars)
print('$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$')
# add selected vars into list
# ('res4' in var.name and ('gamma' in var.name or 'beta' in var.name)) or
for var in tf.global_variables():
if 'resnet_model' in var.name and \
('conv0' in var.name or
'fc' in var.name or
'res3' in var.name or
'res4' in var.name or
'res1' in var.name or
'res2' in var.name) and \
('gamma' in var.name or
'beta' in var.name or
'kernel' in var.name or
'bias' in var.name):
if var not in train_vars:
train_vars.append(var)
elif 'enc_layer' in var.name and \
('kernel' in var.name or
'bias' in var.name or
'gamma' in var.name or
'beta' in var.name):
if var not in train_vars:
train_vars.append(var)
print('$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$')
print_var_list('train_vars', train_vars)
print('$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$')
######################################
# Create a saver.
saver = tf.train.Saver(var_list=tf.all_variables(), max_to_keep=1000)
# start a session with memory growth
config = tf.ConfigProto(log_device_placement=False)
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
print("session created")
# list all checkpoints in ckpt_path
checkpoint_path_read = os.path.join(os.getcwd() +
test_params.check_point_dir)
ckpts = tf.train.get_checkpoint_state(checkpoint_path_read)
print(ckpts)
# find the ckpt we need to load and load it
for ckpt in ckpts.all_model_checkpoint_paths:
# print(ckpt)
ckpt_step = int(ckpt.split('-')[-1])
if ckpt_step == test_params.step_to_load:
saver.restore(sess, ckpt)
print('model loaded from {}'.format(ckpt))
# #######################################
# # setup all attacks
attack_switch = {
'fgsm': False,
'ifgsm': True,
'deepfool': False,
'mim': True,
'spsa': False,
'cwl2': False,
'madry': True,
'stm': False
}
ch_model_probs = CustomCallableModelWrapper(
callable_fn=inference_test_output_probs,
output_layer='probs',
keep_prob=keep_prob,
pre_define_vars=pre_define_vars,
resnet_params=resnet_params,
train_params=train_params)
attack_tensor_testing_dict = {}
# define each attack method's tensor
mu_alpha = tf.placeholder(tf.float32, [1])
# Iterative FGSM (BasicIterativeMethod/ProjectedGradientMethod with no random init)
with tf.device('/gpu:0'):
if attack_switch['ifgsm']:
print('creating attack tensor of BasicIterativeMethod')
ifgsm_obj = BasicIterativeMethod(model=ch_model_probs,
sess=sess)
attack_tensor_testing_dict['ifgsm'] = ifgsm_obj.generate(
x=x_sb,
eps=mu_alpha,
eps_iter=mu_alpha / train_params.iter_step_testing,
nb_iter=train_params.iter_step_testing,
clip_min=-1.0,
clip_max=1.0)
# MomentumIterativeMethod
with tf.device('/gpu:0'):
if attack_switch['mim']:
print('creating attack tensor of MomentumIterativeMethod')
mim_obj = MomentumIterativeMethod(model=ch_model_probs,
sess=sess)
attack_tensor_testing_dict['mim'] = mim_obj.generate(
x=x_sb,
eps=mu_alpha,
eps_iter=mu_alpha / train_params.iter_step_testing,
nb_iter=train_params.iter_step_testing,
decay_factor=1.0,
clip_min=-1.0,
clip_max=1.0)
# MadryEtAl (Projected Grdient with random init, same as rand+fgsm)
with tf.device('/gpu:0'):
if attack_switch['madry']:
print('creating attack tensor of MadryEtAl')
madry_obj = MadryEtAl(model=ch_model_probs, sess=sess)
attack_tensor_testing_dict['madry'] = madry_obj.generate(
x=x_sb,
eps=mu_alpha,
eps_iter=mu_alpha / train_params.iter_step_testing,
nb_iter=train_params.iter_step_testing,
clip_min=-1.0,
clip_max=1.0)
# #######################################
sigmaEGM = all_params['sigmaEGM']
__noiseE = all_params['__noiseE']
grad_redis = all_params['grad_redis']
_sensitivity_2 = all_params['_sensitivity_2']
_sensitivityW = all_params['_sensitivityW']
Delta_redis = all_params['Delta_redis']
sigmaHGM = all_params['sigmaHGM']
__noiseH = all_params['__noiseH']
__noise_zero = all_params['__noise_zero']
####################################
####################################
print('start testing')
start_time = time.time()
log_file_path = os.getcwd() + test_params.log_file_path
log_file = open(log_file_path, 'a', encoding='utf-8')
attacks_and_benign = test_params.attacks + ['benign']
#===================adv samples=====================
# for each eps setting
for fgsm_eps in test_params.fgsm_eps_list:
adv_acc_dict = {}
robust_adv_acc_dict = {}
robust_adv_utility_dict = {}
log_str = ''
eps_start_time = time.time()
# cover all test data
for i in range(test_params.test_epochs):
test_batch = TIN_data.test.next_batch(
test_params.test_batch_size)
adv_images_dict = {}
# test for each attack
for atk in attacks_and_benign:
start_time = time.time()
if atk not in adv_acc_dict:
adv_acc_dict[atk] = 0.0
robust_adv_acc_dict[atk] = 0.0
robust_adv_utility_dict[atk] = 0.0
if atk == 'benign':
testing_img = test_batch[0]
elif attack_switch[atk]:
# if only one gpu available, generate adv samples in-place
if atk not in adv_images_dict:
adv_images_dict[atk] = sess.run(
attack_tensor_testing_dict[atk],
feed_dict={
x_sb: test_batch[0],
mu_alpha: [fgsm_eps],
keep_prob: 1.0
})
testing_img = adv_images_dict[atk]
else:
continue
print('adv gen time: {}s'.format(time.time() - start_time))
start_time = time.time()
### PixelDP Robustness ###
predictions_form_argmax = np.zeros([
test_params.test_batch_size, train_params.num_classes
])
softmax_predictions = sess.run(
y_softmax_test,
feed_dict={
x_sb: testing_img,
noise: (__noiseE + __noiseH) / 2,
keep_prob: 1.0
})
argmax_predictions = np.argmax(softmax_predictions, axis=1)
for n_draws in range(1, test_params.num_samples + 1):
if n_draws % 100 == 0:
print(
'current draws: {}, avg draw time: {}s'.format(
n_draws,
(time.time() - start_time) / n_draws))
_noiseE = np.random.normal(
0.0, sigmaEGM**2,
train_params.enc_h_size * train_params.enc_h_size *
train_params.enc_filters).astype(np.float32)
_noiseE = np.reshape(_noiseE, [
-1, train_params.enc_h_size,
train_params.enc_h_size, train_params.enc_filters
])
_noise = np.random.normal(
0.0, sigmaHGM**2,
train_params.enc_h_size * train_params.enc_h_size *
train_params.enc_filters).astype(np.float32)
_noise = np.reshape(_noise, [
-1, train_params.enc_h_size,
train_params.enc_h_size, train_params.enc_filters
]) * grad_redis
for j in range(test_params.test_batch_size):
pred = argmax_predictions[j]
predictions_form_argmax[j, pred] += 1
softmax_predictions = sess.run(
y_softmax_test,
feed_dict={
x_sb:
testing_img,
noise: (__noiseE + __noiseH) / 2 +
(_noiseE + _noise) / 4,
keep_prob:
1.0
})
argmax_predictions = np.argmax(softmax_predictions,
axis=1)
final_predictions = predictions_form_argmax
is_correct = []
is_robust = []
for j in range(test_params.test_batch_size):
is_correct.append(
np.argmax(test_batch[1][j]) == np.argmax(
final_predictions[j]))
robustness_from_argmax = robustnessGGaussian.robustness_size_argmax(
counts=predictions_form_argmax[j],
eta=0.05,
dp_attack_size=fgsm_eps,
dp_epsilon=train_params.dp_epsilon,
dp_delta=0.05,
dp_mechanism='gaussian') / dp_mult
is_robust.append(robustness_from_argmax >= fgsm_eps)
adv_acc_dict[atk] += np.sum(
is_correct) * 1.0 / test_params.test_batch_size
robust_adv_acc_dict[atk] += np.sum([
a and b for a, b in zip(is_robust, is_correct)
]) * 1.0 / np.sum(is_robust)
robust_adv_utility_dict[atk] += np.sum(
is_robust) * 1.0 / test_params.test_batch_size
dt = time.time() - start_time
print('atk test time: {}s'.format(dt), flush=True)
##############################
# average all acc for whole test data
log_str += datetime.now().strftime("%Y-%m-%d_%H:%M:%S\n")
log_str += 'model trained epoch: {}\n'.format(
test_params.epoch_to_test)
log_str += 'fgsm_eps: {}\n'.format(fgsm_eps)
log_str += 'iter_step_testing: {}\n'.format(
test_params.iter_step_testing)
log_str += 'num_samples: {}\n'.format(test_params.num_samples)
for atk in attacks_and_benign:
adv_acc_dict[atk] = adv_acc_dict[atk] / test_params.test_epochs
robust_adv_acc_dict[
atk] = robust_adv_acc_dict[atk] / test_params.test_epochs
robust_adv_utility_dict[atk] = robust_adv_utility_dict[
atk] / test_params.test_epochs
# added robust prediction
log_str += " {}: {:.6f} {:.6f} {:.6f} {:.6f}\n".format(
atk, adv_acc_dict[atk], robust_adv_acc_dict[atk],
robust_adv_utility_dict[atk],
robust_adv_acc_dict[atk] * robust_adv_utility_dict[atk])
dt = time.time() - eps_start_time
print('total test time: {}s'.format(dt), flush=True)
print(log_str, flush=True)
print('*******************')
log_file.write(log_str)
log_file.write('*******************\n')
log_file.flush()
dt = time.time() - start_time
log_file.close()
def PDP_SSGD_resnet_with_pretrain(TIN_data, resnet_params, train_params,
params_to_save):
# dict for encoding layer variables and output layer variables
pre_define_vars = {}
# list of variables to train
train_vars = []
with tf.Graph().as_default(), tf.device('/cpu:0'):
global_step = tf.Variable(0, trainable=False)
# Parameters Declarification
######################################
# encoding (pretrain) layer variables
with tf.variable_scope('enc_layer', reuse=tf.AUTO_REUSE) as scope:
kernel1 = tf.get_variable(
'kernel1',
shape=[
train_params.enc_kernel_size, train_params.enc_kernel_size,
3, train_params.enc_filters
],
dtype=tf.float32,
initializer=tf.contrib.layers.xavier_initializer_conv2d())
biases1 = tf.get_variable('biases1',
shape=[train_params.enc_filters],
dtype=tf.float32,
initializer=tf.constant_initializer(0.0))
pre_define_vars['kernel1'] = kernel1
pre_define_vars['biases1'] = biases1
train_vars.append(kernel1)
train_vars.append(biases1)
shape = kernel1.get_shape().as_list()
w_t = tf.reshape(kernel1, [-1, shape[-1]])
w = tf.transpose(w_t)
sing_vals = tf.svd(w, compute_uv=False)
sensitivityW = tf.reduce_max(sing_vals)
dp_mult = train_params.attack_norm_bound * math.sqrt(2 * math.log(
1.25 / train_params.dp_delta)) / train_params.dp_epsilon
params_to_save['dp_mult'] = dp_mult
# output layer variables
with tf.variable_scope('fc2', reuse=tf.AUTO_REUSE) as scope:
stdv = 1.0 / math.sqrt(train_params.hk)
final_w = tf.get_variable(
'kernel',
shape=[train_params.hk, train_params.num_classes],
dtype=tf.float32,
initializer=tf.random_uniform_initializer(-stdv, stdv))
final_b = tf.get_variable('bias',
shape=[train_params.num_classes],
dtype=tf.float32,
initializer=tf.constant_initializer(0.0))
pre_define_vars['final_w'] = final_w
pre_define_vars['final_b'] = final_b
train_vars.append(final_w)
train_vars.append(final_b)
######################################
# Build a Graph that computes the logits predictions from the inputs
######################################
# input placeholders
x_sb = tf.placeholder(
tf.float32,
[None, train_params.image_size, train_params.image_size, 3],
name='x_sb') # input is the bunch of n_batchs
x_test = tf.placeholder(
tf.float32,
[None, train_params.image_size, train_params.image_size, 3],
name='x_test')
y_sb = tf.placeholder(
tf.float32, [None, train_params.num_classes],
name='y_sb') # input is the bunch of n_batchs (super batch)
y_test = tf.placeholder(tf.float32, [None, train_params.num_classes],
name='y_test')
noise = tf.placeholder(tf.float32, [
None, train_params.enc_h_size, train_params.enc_h_size,
train_params.enc_filters
],
name='noise') # one time
learning_rate = tf.placeholder(tf.float32,
shape=(),
name='learning_rate')
keep_prob = tf.placeholder(tf.float32, shape=(), name='keep_prob')
# list of grads for each GPU
tower_pretrain_grads = []
tower_train_grads = []
all_train_loss = []
# optimizers
train_opt = tf.train.GradientDescentOptimizer(learning_rate)
with tf.device('/gpu:0'):
# the model
y_logits, h_conv1 = inference(
x_sb, train_params.attack_norm_bound * noise, keep_prob,
pre_define_vars, resnet_params, train_params)
y_softmax = tf.nn.softmax(y_logits)
# loss
with tf.device('/cpu:0'):
loss = lossDPSGD(y_logits, y_sb)
# get total loss tensor for train ops
total_loss = tf.reduce_sum(loss)
# noise redistribution parameters
grad, = tf.gradients(loss, h_conv1)
print(loss)
print(h_conv1)
print(grad)
normalized_grad = tf.sign(grad)
normalized_grad = tf.stop_gradient(normalized_grad)
normalized_grad_r = tf.abs(
tf.reduce_mean(normalized_grad, axis=(0)))**2
sum_r = tf.reduce_sum(normalized_grad_r,
axis=(0, 1, 2),
keepdims=False)
normalized_grad_r = train_params.enc_h_size * train_params.enc_h_size * train_params.enc_filters * normalized_grad_r / sum_r
shape_grad = normalized_grad_r.get_shape().as_list()
grad_t = tf.reshape(normalized_grad_r, [-1, shape_grad[-1]])
g = tf.transpose(grad_t)
sing_g_vals = tf.svd(g, compute_uv=False)
sensitivity_2 = tf.reduce_max(sing_g_vals)
y_logits_test, _ = test_inference(
x_sb, train_params.attack_norm_bound * noise, keep_prob,
pre_define_vars, resnet_params, train_params)
y_softmax_test = tf.nn.softmax(y_logits_test)
correct_prediction = tf.equal(tf.argmax(y_logits_test, 1),
tf.argmax(y_sb, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
priv_accountant = accountant.GaussianMomentsAccountant(
train_params.train_size)
privacy_accum_op = priv_accountant.accumulate_privacy_spending(
[None, None], train_params.sigma, train_params.batch_size)
# print all variables
print('$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$')
all_vars = tf.global_variables()
print_var_list('all vars', all_vars)
print('$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$')
# add selected vars into list
# ('res4' in var.name and ('gamma' in var.name or 'beta' in var.name)) or
for var in tf.global_variables():
if 'resnet_model' in var.name and \
('conv0' in var.name or
'fc' in var.name or
'res3' in var.name or
'res4' in var.name or
'res1' in var.name or
'res2' in var.name) and \
('gamma' in var.name or
'beta' in var.name or
'kernel' in var.name or
'bias' in var.name):
if var not in train_vars:
train_vars.append(var)
elif 'enc_layer' in var.name and \
('kernel' in var.name or
'bias' in var.name or
'gamma' in var.name or
'beta' in var.name):
if var not in train_vars:
train_vars.append(var)
print_var_list('train_vars', train_vars)
print('$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$')
# op for compute grads
with tf.device('/gpu:0'):
# get all update_ops (updates of moving averageand std) for batch normalizations
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
print_op_list('update ops', update_ops)
# compute each gradient and apply clip and noise
train_grads = []
for var in train_vars:
g = tf.clip_by_norm(
tf.gradients(loss, var)[0], train_params.clip_bound)
g = g + tf.random_normal(shape=tf.shape(g),
mean=0.0,
stddev=train_params.sigma *
(train_params.sensitivity**2),
dtype=tf.float32)
train_grads.append((g, var))
# now the train and pretrain ops do not come with update_ops
with tf.control_dependencies(update_ops):
train_op = train_opt.apply_gradients(train_grads,
global_step=global_step)
######################################
# Create a saver.
saver = tf.train.Saver(var_list=tf.all_variables(), max_to_keep=1000)
# start a session with memory growth
config = tf.ConfigProto(log_device_placement=False)
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
print("session created")
# init op
print('initialize_all_variables')
init = tf.initialize_all_variables()
sess.run(init)
if train_params.load_weights:
# load pretrained variables
# load variable convert table
tgt_var_name_dict = {}
with open(train_params.weight_table_path, 'r',
encoding='utf-8') as inf:
lines = inf.readlines()
for line in lines:
var_names = line.strip().split(' ')
if var_names[1] == 'NONE':
continue
else:
tgt_var_name_dict[var_names[0]] = var_names[1]
# load variables
pretrained_var_dict = load_pretrained_vars()
# load pre-trained vars
for var in tf.global_variables():
if var.name in tgt_var_name_dict:
# print('var \"{}\" found'.format(var.name))
try:
var.load(
pretrained_var_dict[tgt_var_name_dict[var.name]],
session=sess)
print('{} loaded'.format(var.name))
except:
print('var {} not loaded since shape changed'.format(
var.name))
else:
if 'Adam' not in var.name:
print('var \"{}\" NOT FOUND'.format(var.name))
else:
print('Training model from scratch')
#####################################
# init noise
s = math.log(sqrt(2.0 / math.pi) * 1e+5)
sigmaEGM = sqrt(2.0) * 1.0 * (sqrt(s) +
sqrt(s + train_params.dp_epsilon)) / (
2.0 * train_params.dp_epsilon)
params_to_save['sigmaEGM'] = sigmaEGM
__noiseE = np.random.normal(
0.0, sigmaEGM**2,
train_params.enc_h_size * train_params.enc_h_size *
train_params.enc_filters).astype(np.float32)
__noiseE = np.reshape(__noiseE, [
-1, train_params.enc_h_size, train_params.enc_h_size,
train_params.enc_filters
])
params_to_save['__noiseE'] = __noiseE
_batch = TIN_data.train.next_super_batch(5,
ensemble=False,
random=True)
grad_redis = sess.run([normalized_grad_r],
feed_dict={
x_sb: _batch[0],
y_sb: _batch[1],
noise: __noiseE * 0,
keep_prob: 1.0
})
params_to_save['grad_redis'] = grad_redis
_sensitivity_2 = sess.run([sensitivity_2],
feed_dict={
x_sb: _batch[0],
y_sb: _batch[1],
noise: __noiseE * 0,
keep_prob: 1.0
})
params_to_save['_sensitivity_2'] = _sensitivity_2
_sensitivityW = sess.run(sensitivityW)
params_to_save['_sensitivityW'] = _sensitivityW
Delta_redis = _sensitivityW / sqrt(_sensitivity_2[0])
params_to_save['Delta_redis'] = Delta_redis
sigmaHGM = sqrt(2.0) * Delta_redis * (
sqrt(s) + sqrt(s + train_params.dp_epsilon)) / (
2.0 * train_params.dp_epsilon)
params_to_save['sigmaHGM'] = sigmaHGM
__noiseH = np.random.normal(
0.0, sigmaHGM**2,
train_params.enc_h_size * train_params.enc_h_size *
train_params.enc_filters).astype(np.float32)
__noiseH = np.reshape(__noiseH, [
-1, train_params.enc_h_size, train_params.enc_h_size,
train_params.enc_filters
]) * grad_redis
params_to_save['__noiseH'] = __noiseH
__noise_zero = np.random.normal(
0.0, 0, train_params.enc_h_size * train_params.enc_h_size *
train_params.enc_filters).astype(np.float32)
__noise_zero = np.reshape(__noise_zero, [
-1, train_params.enc_h_size, train_params.enc_h_size,
train_params.enc_filters
])
params_to_save['__noise_zero'] = __noise_zero
# save params
with open(os.getcwd() + train_params.params_save_path, 'wb') as outf:
pickle.dump(params_to_save, outf)
print('params saved')
####################################
####################################
print('start train')
start_time = time.time()
lr_schedule_list = sorted(train_params.lr_schedule.keys())
# train whole model
step = 0
while True:
# if enough steps, break
if step > train_params.train_steps:
break
# add steps here so not forgot
else:
step += 1
# manual schedule learning rate
current_epoch = step // (train_params.epoch_steps)
current_lr = train_params.lr_schedule[get_lr(
current_epoch, lr_schedule_list)]
# benign and adv batch
super_batch = TIN_data.train.next_super_batch(1,
ensemble=False,
random=True)
_, loss_value = sess.run(
[train_op, loss],
feed_dict={
x_sb: super_batch[0],
y_sb: super_batch[1],
noise: (__noiseE + __noiseH) / 2,
keep_prob: train_params.keep_prob,
learning_rate: current_lr
})
# check with the privacy budget
sess.run([privacy_accum_op])
spent_eps_deltas = priv_accountant.get_privacy_spent(
sess, target_eps=train_params.target_eps)
if step % int(train_params.epoch_steps) == 0:
print(spent_eps_deltas)
_break = False
for _eps, _delta in spent_eps_deltas:
if _delta >= train_params.delta:
_break = True
break
if _break == True:
print('budget all spent, stop training')
break
# print status every epoch
if step % int(train_params.epoch_steps) == 0:
dt = time.time() - start_time
avg_epoch_time = dt / (step / train_params.epoch_steps)
print('epoch: {:.4f}, avg epoch time: {:.4f}s, current_lr: {}'.
format(step / train_params.epoch_steps, avg_epoch_time,
current_lr),
flush=True)
print('train_loss: {:.6f}'.format(loss_value))
# save model
if step % int(train_params.epoch_steps) == 0 and int(
step /
train_params.epoch_steps) in train_params.epochs_to_save:
print('saving model at epoch {}'.format(
step / train_params.epoch_steps))
checkpoint_path = os.path.join(
os.getcwd() + train_params.check_point_dir, 'model.ckpt')
saver.save(sess, checkpoint_path, global_step=step)
# for testing, I use basic inference procedure and accuracy measure on only benign data
# if step % int(train_params.epoch_steps) == 0 and int(step / train_params.epoch_steps) in train_params.epochs_to_test:
if step % int(10 * train_params.epoch_steps) == 0 and step > 0:
test_start = time.time()
print('train test reported at step: {}, epoch: {}'.format(
step, step / train_params.epoch_steps))
dt = time.time() - start_time
avg_epoch_time = dt / (step / train_params.epoch_steps)
print('epoch: {:.4f}, avg epoch time: {:.4f}s, current_lr: {}'.
format(step / train_params.epoch_steps, avg_epoch_time,
current_lr),
flush=True)
print('train_loss: {:.6f}'.format(loss_value))
# print('output layer: \n\t{}'.format(output_layer_value))
#===================adv samples=====================
adv_acc = 0.0
robust_adv_acc = 0.0
robust_adv_utility = 0.0
log_str = ''
# cover all test data
for i in range(train_params.test_epochs):
# start testing
test_batch = TIN_data.test.next_batch(
train_params.test_batch_size)
### PixelDP Robustness ###
predictions_form_argmax = np.zeros([
train_params.test_batch_size, train_params.num_classes
])
softmax_predictions = sess.run(
y_softmax_test,
feed_dict={
x_sb: test_batch[0],
noise: (__noiseE + __noiseH) / 2,
keep_prob: 1.0
})
argmax_predictions = np.argmax(softmax_predictions, axis=1)
for n_draws in range(0, train_params.num_samples):
_noiseE = np.random.normal(
0.0, sigmaEGM**2,
train_params.enc_h_size * train_params.enc_h_size *
train_params.enc_filters).astype(np.float32)
_noiseE = np.reshape(_noiseE, [
-1, train_params.enc_h_size,
train_params.enc_h_size, train_params.enc_filters
])
_noise = np.random.normal(
0.0, sigmaHGM**2,
train_params.enc_h_size * train_params.enc_h_size *
train_params.enc_filters).astype(np.float32)
_noise = np.reshape(_noise, [
-1, train_params.enc_h_size,
train_params.enc_h_size, train_params.enc_filters
]) * grad_redis
for j in range(train_params.test_batch_size):
pred = argmax_predictions[j]
predictions_form_argmax[j, pred] += 1
softmax_predictions = sess.run(
y_softmax_test,
feed_dict={
x_sb:
test_batch[0],
noise: (__noiseE + __noiseH) / 2 +
(_noiseE + _noise) / 4,
keep_prob:
1.0
})
argmax_predictions = np.argmax(softmax_predictions,
axis=1)
final_predictions = predictions_form_argmax
is_correct = []
is_robust = []
for j in range(train_params.test_batch_size):
is_correct.append(
np.argmax(test_batch[1][j]) == np.argmax(
final_predictions[j]))
robustness_from_argmax = robustnessGGaussian.robustness_size_argmax(
counts=predictions_form_argmax[j],
eta=0.05,
dp_attack_size=train_params.fgsm_eps,
dp_epsilon=train_params.dp_epsilon,
dp_delta=0.05,
dp_mechanism='gaussian') / dp_mult
is_robust.append(
robustness_from_argmax >= train_params.fgsm_eps)
adv_acc += np.sum(
is_correct) * 1.0 / train_params.test_batch_size
robust_adv_acc += np.sum([
a and b for a, b in zip(is_robust, is_correct)
]) * 1.0 / np.sum(is_robust)
robust_adv_utility += np.sum(
is_robust) * 1.0 / train_params.test_batch_size
##############################
# average all acc for whole test data
adv_acc = adv_acc / train_params.test_epochs
robust_adv_acc = robust_adv_acc / train_params.test_epochs
robust_adv_utility = robust_adv_utility / train_params.test_epochs
log_str += " {}: {:.6f} {:.6f} {:.6f} {:.6f}\n".format(
'benign', adv_acc, robust_adv_acc, robust_adv_utility,
robust_adv_acc * robust_adv_utility)
dt = time.time() - test_start
print('testing time: {}'.format(dt))
print(log_str, flush=True)
print('*******************')