def run_adversarial_training(cfg):
'''
Algorithm 1 in the paper
'''
def run_validation(input_op_list, summary_file, summary_info):
'''
Validation during training.
Validation can be run on any set: training, validating or testing.
Input:
sess: run oprations in this session.
input_op_list: list. For example, when validating on training set, it is [tr_videos_op, tr_action_labels_op, tr_actor_labels_op]
other_op_list: list. Always [accuracy_utility, accuracy_budget, loss_utility_op, loss_budget_op]
summary_file: put the validation summary in this file.
summary_info: string. Summary content.
Output:
print and write summary.
Return:
acc_util_lst, acc_budget_lst
'''
# initialize timer and lists:
start_time = time.time()
acc_util_lst, acc_budget_lst, loss_utility_lst, loss_budget_lst = [], [], [], []
# late update:
for _ in itertools.repeat(None, FLAGS.n_minibatches_eval):
tr_videos, tr_action_labels, tr_actor_labels = sess.run(
input_op_list)
acc_util, acc_budget, loss_utility_value, loss_budget_value = sess.run(
[
accuracy_utility, accuracy_budget, loss_utility_op,
loss_budget_op
],
feed_dict={
videos_placeholder: tr_videos,
utility_labels_placeholder: tr_action_labels,
budget_labels_placeholder: tr_actor_labels,
dropout_placeholder: 1.0,
})
acc_util_lst.append(acc_util)
acc_budget_lst.append(acc_budget)
loss_utility_lst.append(loss_utility_value)
loss_budget_lst.append(loss_budget_value)
# Writing and printing summary part:
summary = summary_info.format(time.time() - start_time,
np.mean(acc_util_lst),
np.mean(acc_budget_lst),
np.mean(loss_utility_lst),
np.mean(loss_budget_lst))
print(summary)
summary_file.write(summary + '\n')
print('\n')
# End writing and printing summary part.
return acc_util_lst, acc_budget_lst
# initialize multiplier_lst, logits_budget_lst_dct, loss_budget_lst_dct, which are used in both the graph and the session:
# The depth multiplier list for creating different budget models ensemble (MobileNet with different depth.)
multiplier_lst = [0.60 - i * 0.02 for i in range(FLAGS.NBudget)]
# The dict of logits and loss for each different budget model to get accuracy
logits_budget_lst_dct = {
str(multiplier): []
for multiplier in multiplier_lst
}
loss_budget_lst_dct = {
str(multiplier): []
for multiplier in multiplier_lst
}
# end initializing multiplier_lst, logits_budget_lst_dct, loss_budget_lst_dct.
# mkdir for saving ckpt of the adversarial training process:
if not os.path.exists(ckpt_dir):
os.makedirs(ckpt_dir)
# define graph
graph = tf.Graph()
with graph.as_default():
# global step:
global_step = tf.get_variable('global_step', [],
initializer=tf.constant_initializer(0),
trainable=False)
# placeholder inputs:
videos_placeholder, utility_labels_placeholder, budget_labels_placeholder, dropout_placeholder, _ = \
placeholder_inputs(cfg['TRAIN']['BATCH_SIZE'] * FLAGS.GPU_NUM, cfg)
tower_grads_degrad, tower_grads_utility, tower_grads_budget = [], [], []
# Compute Acc (fT, fb logits output)
logits_utility_lst, logits_budget_lst = [], []
# Compute Loss (LT, Lb_cross_entropy, Ld=LT+Lb_entropy?)
loss_utility_lst, loss_budget_lst, loss_degrad_lst = [], [], []
# Compute prediction with min entropy (most confident)
# Use max uniform loss instead
argmax_adverse_budget_loss_lst = []
# Optimizer for the 3 components respectively
opt_degrad = tf.train.AdamOptimizer(FLAGS.degradation_lr)
opt_utility = tf.train.AdamOptimizer(FLAGS.utility_lr)
opt_budget = tf.train.AdamOptimizer(FLAGS.budget_lr)
with tf.variable_scope(tf.get_variable_scope()):
for gpu_index in range(0, FLAGS.GPU_NUM):
with tf.device('/gpu:%d' % gpu_index):
print('/gpu:%d' % gpu_index)
with tf.name_scope('%s_%d' % ('gpu', gpu_index)) as scope:
videos = videos_placeholder[gpu_index *
cfg['TRAIN']['BATCH_SIZE']:
(gpu_index + 1) *
cfg['TRAIN']['BATCH_SIZE']]
utility_labels = utility_labels_placeholder[
gpu_index *
cfg['TRAIN']['BATCH_SIZE']:(gpu_index + 1) *
cfg['TRAIN']['BATCH_SIZE']]
budget_labels = budget_labels_placeholder[
gpu_index *
cfg['TRAIN']['BATCH_SIZE']:(gpu_index + 1) *
cfg['TRAIN']['BATCH_SIZE']]
loss_degrad, loss_budget, loss_utility, logits_budget, logits_utility, argmax_adverse_budget_loss = \
create_architecture_adversarial(cfg, cfg['TRAIN']['BATCH_SIZE'], multiplier_lst, logits_budget_lst_dct, loss_budget_lst_dct, scope, videos, utility_labels, budget_labels, dropout_placeholder)
# Reuse variables for the next tower.
tf.get_variable_scope().reuse_variables()
loss_degrad_lst.append(loss_degrad)
loss_budget_lst.append(loss_budget)
loss_utility_lst.append(loss_utility)
logits_budget_lst.append(logits_budget)
logits_utility_lst.append(logits_utility)
argmax_adverse_budget_loss_lst.append(
argmax_adverse_budget_loss)
# varlist:
varlist_degrad = [
v for v in tf.trainable_variables()
if any(x in v.name for x in ["DegradationModule"])
]
varlist_utility = [
v for v in tf.trainable_variables()
if any(x in v.name for x in ["UtilityModule"])
]
varlist_budget = [
v for v in tf.trainable_variables()
if any(x in v.name for x in ["BudgetModule"])
]
grads_degrad = opt_degrad.compute_gradients(
loss_degrad, varlist_degrad)
grads_budget = opt_budget.compute_gradients(
loss_budget, varlist_budget)
grads_utility = opt_utility.compute_gradients(
loss_utility, varlist_utility + varlist_degrad)
tower_grads_degrad.append(grads_degrad)
tower_grads_budget.append(grads_budget)
tower_grads_utility.append(grads_utility)
# argmax_adverse_budget_loss_op = tf.concat(argmax_adverse_budget_loss_lst, 0)
# Average losses over each GPU:
loss_utility_op = tf.reduce_mean(loss_utility_lst,
name='softmax') # LT
loss_budget_op = tf.reduce_mean(loss_budget_lst, name='softmax') # Lb
loss_degrad_op = tf.reduce_mean(loss_degrad_lst,
name='softmax') # Ld = -Lb
# Concatenate the logits over all GPU:
logits_utility = tf.concat(logits_utility_lst, 0)
logits_budget = tf.concat(logits_budget_lst, 0)
# acc
accuracy_utility = accuracy(logits_utility, utility_labels_placeholder)
accuracy_budget = accuracy(logits_budget, budget_labels_placeholder)
# count how many testing samples are classified correctly:
right_count_utility_op = correct_num(logits_utility,
utility_labels_placeholder)
right_count_budget_op = correct_num(logits_budget,
budget_labels_placeholder)
# operations on each budget model:
loss_budget_op_lst = []
accuracy_budget_list = []
right_count_budget_op_lst = []
# for each mobile-net:
for multiplier in multiplier_lst: # multiplier_lst has M elements -> each is the channel depth of a mobile net.
# loss of each model:
loss_budget_op_each_model = tf.reduce_mean(
loss_budget_lst_dct[str(multiplier)]
) # mean loss over multi-gpu of a certain mobile-net.
loss_budget_op_lst.append(loss_budget_op_each_model)
# logits of each model:
budget_logits_each_model = tf.concat(
logits_budget_lst_dct['{}'.format(multiplier)],
0) # same budget model, concatenate over GPUs.
# acc of each model
accuracy_budget_each_model = accuracy(budget_logits_each_model,
budget_labels_placeholder)
accuracy_budget_list.append(accuracy_budget_each_model)
# right count of each model:
right_count_op = correct_num(budget_logits_each_model,
budget_labels_placeholder)
right_count_budget_op_lst.append(right_count_op)
'''
The only thing changed here is tower_grads_degrad, the second parameter of the create_grad_accum_for_late_update function.
The trainable variable list needn't be changed, it is still varlist_degrad, which is the variables in fd network.
The loss is changed:
zero_ops_degrad, accum_ops_degrad, apply_gradient_op_degrad <- tower_grads_degrad <- grads_degrade <- loss_degrad
So, we only need to change the definition of 'loss_degrad' in 'create_architecture_adversarial' function in main_import.py.
'''
zero_ops_degrad, accum_ops_degrad, apply_gradient_op_degrad = create_grad_accum_for_late_update(
opt_degrad,
tower_grads_degrad,
varlist_degrad,
global_step,
decay_with_global_step=True)
zero_ops_budget, accum_ops_budget, apply_gradient_op_budget = create_grad_accum_for_late_update(
opt_budget,
tower_grads_budget,
varlist_budget,
global_step,
decay_with_global_step=False)
zero_ops_utility, accum_ops_utility, apply_gradient_op_utility = create_grad_accum_for_late_update(
opt_utility,
tower_grads_utility,
varlist_utility + varlist_degrad,
global_step,
decay_with_global_step=False)
tr_videos_op, tr_action_labels_op, tr_actor_labels_op = create_videos_reading_ops(
is_train=True, is_val=False, cfg=cfg)
val_videos_op, val_action_labels_op, val_actor_labels_op = create_videos_reading_ops(
is_train=False, is_val=True, cfg=cfg)
# session config:
config = tf.ConfigProto(allow_soft_placement=True,
log_device_placement=False)
config.gpu_options.allow_growth = True
# run session:
with tf.Session(graph=graph, config=config) as sess:
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
init_op = tf.group(tf.local_variables_initializer(),
tf.global_variables_initializer())
sess.run(init_op)
# load ckpts:
if use_pretrained_model: # load ckpts from pretrained fd and fT.(By run_pretraining_degrad and run_pretraining_utility functions.)
# fT and fd part:
restore_model_ckpt(
sess, FLAGS.deg_target_models, varlist_utility + varlist_degrad
) # FLAGS.deg_target_models is the dir storing ckpt of theta_T and theta_d
# fb part:
restore_model_ckpt(sess, FLAGS.budget_models, varlist_budget)
else: # load ckpts from previous training stage of this run_adversarial_training function.
saver = tf.train.Saver(tf.trainable_variables())
ckpt = tf.train.get_checkpoint_state(checkpoint_dir=ckpt_dir)
if ckpt and ckpt.model_checkpoint_path:
saver.restore(sess, ckpt.model_checkpoint_path)
print('Session restored from trained model at {}!'.format(
ckpt.model_checkpoint_path))
else:
raise FileNotFoundError(errno.ENOENT,
os.strerror(errno.ENOENT), ckpt_dir)
# saver and summary files:
saver = tf.train.Saver(var_list=tf.trainable_variables(),
max_to_keep=10)
loss_summary_file, validation_train_set_summary_file, validation_val_set_summary_file, model_restarting_summary_file = create_summary_files(
summary_dir)
# Adversarial training loop:
for step in xrange(cfg['TRAIN']['TOP_MAXSTEP']):
# Part 0: Model restarting
if (step != 0 and
(FLAGS.use_restarting and step % FLAGS.restarting_step == 0)):
# reinitialize fb:
budget_varlist = [
v for v in tf.trainable_variables()
if any(x in v.name for x in ["BudgetModule"])
]
init_budget_op = tf.variables_initializer(budget_varlist)
sess.run(init_budget_op)
# finish reinitializing fb
# Train theta_B using Lb(X,Y_B) for FLAGS.retraining_step steps:
for Restarting_step in range(0, FLAGS.retraining_step):
start_time = time.time()
acc_util_lst, acc_budget_lst, loss_degrad_lst, loss_utility_lst, loss_budget_lst = [], [], [], [], []
sess.run(zero_ops_budget)
# accumulating gradient for late update:
for _ in itertools.repeat(None, 20):
# placeholder inputs:
tr_videos, tr_action_labels, tr_actor_labels = sess.run(
[
tr_videos_op, tr_action_labels_op,
tr_actor_labels_op
])
# run operations:
_, acc_util, acc_budget, loss_degrad_value, loss_utility_value, loss_budget_value = sess.run(
[
accum_ops_budget, accuracy_utility,
accuracy_budget, loss_degrad_op,
loss_utility_op, loss_budget_op
],
feed_dict={
videos_placeholder: tr_videos,
utility_labels_placeholder: tr_action_labels,
budget_labels_placeholder: tr_actor_labels,
dropout_placeholder: 1.0,
})
# append:
acc_util_lst.append(acc_util)
acc_budget_lst.append(acc_budget)
loss_degrad_lst.append(loss_degrad_value)
loss_utility_lst.append(loss_utility_value)
loss_budget_lst.append(loss_budget_value)
# finish accumulating gradient for late update
# after accumulating gradient, do the update on fb:
sess.run(apply_gradient_op_budget)
# finish update on fb
assert not np.isnan(np.mean(
loss_degrad_lst)), 'Model diverged with loss = NaN'
# loss summary:
loss_summary = 'Restarting (Budget), Step: {:4d}, Restarting_step: {:4d}, time: {:.4f}, budget loss: {:.8f}, ' \
'training budget accuracy: {:.5f}, utility loss: {:.8f}, training utility accuracy: {:.5f}'.format(
step, Restarting_step, time.time() - start_time,
np.mean(loss_budget_lst), np.mean(acc_budget_lst), np.mean(loss_utility_lst), np.mean(acc_util_lst))
model_restarting_summary_file.write(loss_summary + '\n')
print(loss_summary)
# end of loss summary
# finish training theta_B using Lb(X,Y_B) for FLAGS.retraining_step steps.
print('')
loss_summary_file.write('\n')
# End part 0
# Part 3: train Fb using L_b (cross entropy)
for L_b_step in range(0, cfg['TRAIN']['L_B_MAXSTEP']):
# max step: optimize theta_d using L_b(X,Y_B)
for L_b_max_step in range(0,
cfg['TRAIN']['L_B_MAX_PART_STEP']):
start_time = time.time()
acc_util_lst, acc_budget_lst, loss_utility_lst, loss_budget_lst = [], [], [], []
sess.run(zero_ops_degrad)
# accumulating gradient for late update:
for _ in itertools.repeat(None, FLAGS.n_minibatches):
# placeholder inputs:
tr_videos, tr_action_labels, tr_actor_labels = sess.run(
[
tr_videos_op, tr_action_labels_op,
tr_actor_labels_op
])
# run operations:
_, acc_util, acc_budget, loss_utility_value, loss_budget_value = sess.run(
[
accum_ops_degrad, accuracy_utility,
accuracy_budget, loss_utility_op,
loss_budget_op
],
feed_dict={
videos_placeholder: tr_videos,
utility_labels_placeholder: tr_action_labels,
budget_labels_placeholder: tr_actor_labels,
dropout_placeholder: 1.0,
})
# append loss and acc for budget model:
acc_util_lst.append(acc_util)
acc_budget_lst.append(acc_budget)
loss_utility_lst.append(loss_utility_value)
loss_budget_lst.append(loss_budget_value)
# finish accumulating gradient for late update
# after accumulating gradient, do the update on fd:
_ = sess.run([apply_gradient_op_degrad])
# finish update on fd
assert not np.isnan(np.mean(
loss_budget_value)), 'Model diverged with loss = NaN'
# loss summary:
if L_b_max_step % cfg['TRAIN']['L_B_MAX_PRINT_STEP'] == 0:
loss_summary = 'Alternating Training (Budget L_b MAX), Step: {:4d}, L_b_step: {:4d}, L_b_max_step: {:4d} time: {:.4f}, ' \
'training utility accuracy: {:.5f}, training budget accuracy: {:.5f}, ' \
'utility loss: {:.8f}, budget loss: {:.8f}'.format(
step, L_b_step, L_b_max_step, time.time() - start_time,
np.mean(acc_util_lst), np.mean(acc_budget_lst),
np.mean(loss_utility_lst), np.mean(loss_budget_lst)
)
print(loss_summary)
loss_summary_file.write(loss_summary + '\n')
# end loss summary
print()
# End max step
# min step: optimize theta_b using L_b(X,Y_B)
for L_b_min_step in range(0,
cfg['TRAIN']['L_B_MIN_PART_STEP']):
start_time = time.time()
sess.run(zero_ops_budget)
acc_budget_lst, loss_budget_lst = [], []
# accumulating gradient for late update:
for _ in itertools.repeat(None, FLAGS.n_minibatches):
# placeholder inputs:
tr_videos, tr_actor_labels = sess.run(
[tr_videos_op, tr_actor_labels_op])
# run operations:
_, acc_budget, loss_budget_value = sess.run(
[
accum_ops_budget, accuracy_budget,
loss_budget_op
],
feed_dict={
videos_placeholder: tr_videos,
# utility_labels_placeholder: tr_action_labels,
budget_labels_placeholder: tr_actor_labels,
dropout_placeholder: 1.0,
})
# append loss and acc for budget model:
acc_budget_lst.append(acc_budget)
loss_budget_lst.append(loss_budget_value)
# finish accumulating gradient for late update
assert not np.isnan(np.mean(
loss_budget_lst)), 'Model diverged with loss = NaN'
# Monitoring fb using training set
if L_b_min_step % cfg['TRAIN']['MONITOR_STEP'] == 0:
if np.mean(acc_budget_lst
) >= FLAGS.highest_budget_acc_val:
print(bcolors.OKGREEN + 'pass budget acc bar!\n' +
bcolors.ENDC)
loss_summary_file.write('pass budget acc bar!\n')
break
# End monitoring fb on training set.
# after accumulating gradient, do the update on fb, if it didn't pass the budget acc bar:
sess.run([apply_gradient_op_budget])
# finish update on fb
# loss summary:
if L_b_min_step % cfg['TRAIN']['MONITOR_STEP'] == 0:
loss_summary = 'Alternating Training (Budget L_b MIN), Step: {:4d}, L_b_step: {:4d}, L_b_min_step: {:4d} time: {:.4f}, ' \
'training budget accuracy: {:.5f}, budget loss: {:.8f}'.format(
step, L_b_step, L_b_min_step, time.time() - start_time,
np.mean(acc_budget_lst), np.mean(loss_budget_lst)
)
print(loss_summary)
loss_summary_file.write(loss_summary + '\n')
# end loss summary
print('')
loss_summary_file.write('\n')
# End part 3
# Part 2: End-to-end train Ft and Fd using L_T
for L_T_step in range(0, cfg['TRAIN']['L_T_MAXSTEP']):
# Monitoring LT using validation set:
if L_T_step % cfg['TRAIN']['MONITOR_STEP'] == 0:
acc_util_lst, _ = run_validation(input_op_list=[val_videos_op, val_action_labels_op, val_actor_labels_op],
summary_file=loss_summary_file,
summary_info="Monitoring L_T:\n" \
"Step: %d, L_T_step: %d, time: {:.4f}, " \
"validation utility accuracy: {:.5f}, validation budget accuracy: {:.5f}, " \
"utility loss: {:.8f}, budget loss: {:.8f}" % (step, L_T_step))
# breaking condition: (if performance on L_T is still good)
if np.mean(acc_util_lst) >= FLAGS.highest_util_acc_val:
print(bcolors.OKGREEN + 'pass utility acc bar!\n' +
bcolors.ENDC)
loss_summary_file.write('pass utility acc bar!\n')
break
# End of monitoring LT
# Optimizing LT (if necessary) using training set: (This is one batch=FLAGS.n_minibatches, each minibatch has FLAGS.GPU_NUM*cfg['TRAIN']['BATCH_SIZE'] video clips.)
start_time = time.time()
sess.run(zero_ops_utility)
acc_util_lst, acc_budget_lst, loss_utility_lst, loss_budget_lst = [], [], [], []
# accumulating gradient for late update:
for _ in itertools.repeat(None, FLAGS.n_minibatches):
tr_videos, tr_action_labels, tr_actor_labels = sess.run([
tr_videos_op, tr_action_labels_op, tr_actor_labels_op
])
_, acc_util, acc_budget, loss_utility_value, loss_budget_value = sess.run(
[
accum_ops_utility, accuracy_utility,
accuracy_budget, loss_utility_op, loss_budget_op
],
feed_dict={
videos_placeholder: tr_videos,
utility_labels_placeholder: tr_action_labels,
budget_labels_placeholder: tr_actor_labels,
dropout_placeholder: 0.5,
})
acc_util_lst.append(acc_util)
acc_budget_lst.append(acc_budget)
loss_utility_lst.append(loss_utility_value)
loss_budget_lst.append(loss_budget_value)
# finish accumulating gradient for late update
# after accumulating gradient, do the update on fT and fd:
sess.run([apply_gradient_op_utility])
# finish update on fT and fd
assert not np.isnan(np.mean(
loss_utility_lst)), 'Model diverged with loss = NaN'
# loss summary:
loss_summary = 'Alternating Training (Utility), Step: {:4d}, L_T_step: {:4d}, time: {:.4f}, ' \
'training utility accuracy: {:.5f}, training budget accuracy: {:.5f}, ' \
'utility loss: {:.8f}, budget loss: {:.8f}'.format(
step, L_T_step, time.time() - start_time,
np.mean(acc_util_lst), np.mean(acc_budget_lst),
np.mean(loss_utility_lst), np.mean(loss_budget_lst)
)
print(loss_summary)
loss_summary_file.write(loss_summary + '\n')
# end of loss summary
# End of optimizing LT.
print('')
loss_summary_file.write('\n')
# End part 2
# Do validation (on training set and validation set):
if step % cfg['TRAIN']['VAL_STEP'] == 0:
run_validation(input_op_list=[tr_videos_op, tr_action_labels_op, tr_actor_labels_op],
summary_file=validation_train_set_summary_file,
summary_info="Validation train_set summary\n" \
"Step: %d, time: {:.4f}, " \
"training utility accuracy: {:.5f}, training budget accuracy: {:.5f}, " \
"utility loss: {:.8f}, budget loss: {:.8f}" % step)
run_validation(input_op_list=[val_videos_op, val_action_labels_op, val_actor_labels_op],
summary_file=validation_val_set_summary_file,
summary_info="Validation val_set summary\n" \
"Step: %d, time: {:.4f}, " \
"validation utility accuracy: {:.5f}, validation budget accuracy: {:.5f}, " \
"utility loss: {:.8f}, budget loss: {:.8f}" % step)
# End evaluation
# Save ckpt for kb_adversarial learning:
if step % cfg['TRAIN']['SAVE_STEP'] == 0 or (
step + 1) == cfg['TRAIN']['TOP_MAXSTEP']:
checkpoint_path = os.path.join(ckpt_dir, 'model.ckpt')
saver.save(sess, checkpoint_path, global_step=step)
# End evaluation
loss_summary_file.close()
validation_train_set_summary_file.close()
validation_val_set_summary_file.close()
coord.request_stop()
coord.join(threads)
print("done")
def run_adversarial_testing(cfg):
'''
Run testing of the trained model (direct test without any retraining, different from the two-fold-evaluation proposed in the paper)
It will give the utility task accuracy and the privacy budget task accuracy
'''
# initialize multiplier_lst, logits_budget_lst_dct, loss_budget_lst_dct, which are used in both the graph and the session:
# The depth multiplier list for creating different budget models ensemble (MobileNet with different depth.)
multiplier_lst = [0.60 - i * 0.02 for i in range(FLAGS.NBudget)]
# The dict of logits and loss for each different budget model to get accuracy
logits_budget_lst_dct = {
str(multiplier): []
for multiplier in multiplier_lst
}
loss_budget_lst_dct = {
str(multiplier): []
for multiplier in multiplier_lst
}
# end initializing multiplier_lst, logits_budget_lst_dct, loss_budget_lst_dct.
graph = tf.Graph()
with graph.as_default():
# placeholder inputs:
videos_placeholder, utility_labels_placeholder, budget_labels_placeholder, dropout_placeholder, _ = placeholder_inputs(
cfg['TEST']['BATCH_SIZE'] * FLAGS.GPU_NUM, cfg)
# Compute Acc
logits_utility_lst, logits_budget_lst = [], []
with tf.variable_scope(tf.get_variable_scope()) as scope:
# get the logits_budget and logits_utility on each gpu:
for gpu_index in range(0, FLAGS.GPU_NUM):
with tf.device('/gpu:%d' % gpu_index):
print('/gpu:%d' % gpu_index)
with tf.name_scope('%s_%d' % ('gpu', gpu_index)) as scope:
videos = videos_placeholder[gpu_index *
cfg['TEST']['BATCH_SIZE']:
(gpu_index + 1) *
cfg['TEST']['BATCH_SIZE']]
utility_labels = utility_labels_placeholder[
gpu_index *
cfg['TEST']['BATCH_SIZE']:(gpu_index + 1) *
cfg['TEST']['BATCH_SIZE']]
budget_labels = budget_labels_placeholder[
gpu_index *
cfg['TEST']['BATCH_SIZE']:(gpu_index + 1) *
cfg['TEST']['BATCH_SIZE']]
_, _, _, logits_budget, logits_utility, _ = create_architecture_adversarial(
cfg, cfg['TEST']['BATCH_SIZE'], multiplier_lst,
logits_budget_lst_dct, loss_budget_lst_dct, scope,
videos, utility_labels, budget_labels,
dropout_placeholder)
logits_budget_lst.append(logits_budget)
logits_utility_lst.append(logits_utility)
# print('len(logits_utility_lst):', len(logits_utility_lst))
tf.get_variable_scope().reuse_variables()
# concatnate the logits of each gpu:
logits_utility = tf.concat(logits_utility_lst, 0)
logits_budget = tf.concat(logits_budget_lst, 0)
# count how many testing samples are classified correctly:
right_count_utility_op = correct_num(logits_utility,
utility_labels_placeholder)
right_count_budget_op = correct_num(logits_budget,
budget_labels_placeholder)
# operations on each budget model:
right_count_budget_op_lst = []
for multiplier in multiplier_lst:
# right count of each model:
budget_logits_each_model = tf.concat(
logits_budget_lst_dct['{}'.format(multiplier)],
0) # same budget model, concatenate over GPUs.
right_count_op = correct_num(budget_logits_each_model,
budget_labels_placeholder)
right_count_budget_op_lst.append(right_count_op)
videos_op, action_labels_op, actor_labels_op = create_videos_reading_ops(
is_train=False, is_val=False, cfg=cfg)
# session config:
config = tf.ConfigProto(allow_soft_placement=True,
log_device_placement=False)
config.gpu_options.allow_growth = True
# run session:
with tf.Session(graph=graph, config=config) as sess:
# initialization:
init_op = tf.group(tf.local_variables_initializer(),
tf.global_variables_initializer())
sess.run(init_op)
# initialization part should be put outside the multi-threads part! But why?
# multi-threads:
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
# Create a saver for loading trained checkpoints:
saver = tf.train.Saver(tf.trainable_variables())
ckpt = tf.train.get_checkpoint_state(checkpoint_dir=ckpt_dir)
# load trained checkpoints:
if ckpt and ckpt.model_checkpoint_path:
saver.restore(sess, ckpt.model_checkpoint_path)
print('Session restored from trained model at {}!'.format(
ckpt.model_checkpoint_path))
else:
raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT),
ckpt_dir)
total_v = 0.0 # total number of testing samples
test_correct_num_utility = 0.0 # how many testing samples get correct utility label prediction.
test_correct_num_budget = 0.0 # how many testing samples get correct budget label prediction.
test_correct_num_budget_lst = [0.0] * FLAGS.NBudget
print('coord.should_stop():', coord.should_stop())
try:
c = 0
batch_size = cfg['TEST']['BATCH_SIZE'] * FLAGS.GPU_NUM
while not coord.should_stop():
c += 1
print('in while loop ', str(c))
# input operations:
test_videos, test_action_labels, test_actor_labels = sess.run(
[videos_op, action_labels_op, actor_labels_op])
total_v += test_action_labels.shape[0]
# padding:
if test_videos.shape[
0] < batch_size: # the last batch of testing data
test_videos = np.pad(
test_videos, ((0, batch_size - test_videos.shape[0]),
(0, 0), (0, 0), (0, 0), (0, 0)),
'constant',
constant_values=0)
test_actor_labels = np.pad(
test_actor_labels,
((0, batch_size - test_actor_labels.shape[0])),
'constant',
constant_values=-1)
test_action_labels = np.pad(
test_action_labels,
((0, batch_size - test_action_labels.shape[0])),
'constant',
constant_values=-1)
# the padded videos will never be true, since it can never be classified as -1
print('test_videos:', test_videos.shape)
print('test_action_labels:', test_action_labels.shape)
print('test_actor_labels:', test_actor_labels.shape)
# placeholders:
feed_dict = {
videos_placeholder: test_videos,
budget_labels_placeholder: test_actor_labels,
utility_labels_placeholder: test_action_labels,
dropout_placeholder: 1.0
}
# feed dorward:
right_counts = sess.run(
[right_count_utility_op, right_count_budget_op] +
right_count_budget_op_lst,
feed_dict=feed_dict)
print('right_counts:', right_counts)
test_correct_num_utility += right_counts[0]
test_correct_num_budget += right_counts[1]
# testing acc for each one of N budget models:
for i in range(FLAGS.NBudget):
test_correct_num_budget_lst[i] += right_counts[i + 2]
# end testing acc for each one of N budget models.
# end try
except tf.errors.OutOfRangeError:
print('Done testing on all the examples')
finally:
coord.request_stop()
# print and write file:
test_result_str = (
'test_acc_utility: {}, test_correct_num_utility: {}, total_v: {}\n'
'test_acc_budget: {}, test_correct_num_budget: {}, total_v: {}\n'
).format(test_correct_num_utility / total_v, test_correct_num_utility,
total_v, test_correct_num_budget / total_v,
test_correct_num_budget, total_v)
print(test_result_str)
if not os.path.exists(test_result_dir):
os.makedirs(test_result_dir)
test_result_file = open(test_result_dir + '/EvaluationResuls.txt', 'w')
# write testing result to file:
test_result_file.write(test_result_str)
for i in range(FLAGS.NBudget):
test_result_file.write(
'Budget{} test acc: {},\ttest_correct_num: {}\t: total_v: {}\n'
.format(multiplier_lst[i],
test_correct_num_budget_lst[i] / total_v,
test_correct_num_budget_lst[i], total_v))
# finish writing testing result to file.
coord.join(threads)
sess.close()
def build_graph(model_name):
'''
Returns:
graph, init_op, train_op,
logits_op, acc_op, correct_count_op, loss_op,
tr_videos_op, tr_actor_labels_op, val_videos_op, val_actor_labels_op, test_videos_op, test_actor_labels_op,
videos_placeholder, labels_placeholder,
varlist_budget, varlist_degrad
'''
graph = tf.Graph()
with graph.as_default():
# global step:
global_step = tf.get_variable('global_step', [], initializer=tf.constant_initializer(0), trainable=False)
# placholder inputs for graph:
videos_placeholder, labels_placeholder, istraining_placeholder = placeholder_inputs(cfg['TRAIN']['BATCH_SIZE'] * FLAGS.GPU_NUM, cfg)
# degradation models:
network_fn = nets_factory.get_network_fn(model_name,
num_classes=cfg['DATA']['NUM_CLASSES'],
weight_decay=cfg['TRAIN']['WEIGHT_DECAY'],
is_training=istraining_placeholder)
# grads, logits, loss list:
tower_grads = []
logits_lst = []
losses_lst = []
# operation method:
opt = tf.train.AdamOptimizer(1e-4)
with tf.variable_scope(tf.get_variable_scope()) as scope:
for gpu_index in range(0, FLAGS.GPU_NUM):
with tf.device('/gpu:%d' % gpu_index):
print('/gpu:%d' % gpu_index)
with tf.name_scope('%s_%d' % ('gpu', gpu_index)) as scope:
videos = videos_placeholder[gpu_index * cfg['TRAIN']['BATCH_SIZE']:(gpu_index + 1) * cfg['TRAIN']['BATCH_SIZE']]
budget_labels = labels_placeholder[gpu_index * cfg['TRAIN']['BATCH_SIZE']:(gpu_index + 1) * cfg['TRAIN']['BATCH_SIZE']]
degrad_videos = residualNet(videos, is_video=True)
degrad_videos = tf.reshape(degrad_videos, [cfg['TRAIN']['BATCH_SIZE'] * cfg['DATA']['DEPTH'], cfg['DATA']['CROP_HEIGHT'], cfg['DATA']['CROP_WIDTH'], cfg['DATA']['NCHANNEL']])
# logits:
logits, _ = network_fn(degrad_videos)
logits = tf.reshape(logits, [-1, cfg['DATA']['DEPTH'], cfg['DATA']['NUM_CLASSES']])
logits = tf.reduce_mean(logits, axis=1, keep_dims=False)
# loss:
loss = tower_loss_xentropy_sparse(scope, logits, budget_labels)
# append list:
logits_lst.append(logits)
losses_lst.append(loss)
# varible list of budget model:
varlist_budget = [v for v in tf.trainable_variables() if
any(x in v.name for x in ["InceptionV1", "InceptionV2",
"resnet_v1_50", "resnet_v1_101", "resnet_v2_50", "resnet_v2_101",
'MobilenetV1'])]
# varible list of degrade model:
varlist_degrad = [v for v in tf.trainable_variables() if v not in varlist_budget]
# append grads:
tower_grads.append(opt.compute_gradients(loss, varlist_budget))
# reuse variables:
tf.get_variable_scope().reuse_variables()
# loss tensor:
loss_op = tf.reduce_mean(losses_lst)
# acc tensor:
logits_op = tf.concat(logits_lst, 0)
acc_op = accuracy(logits_op, labels_placeholder)
# how many is correctly classified:
correct_count_op = tf.reduce_sum(
tf.cast(tf.equal(tf.argmax(tf.nn.softmax(logits_op), axis=1), labels_placeholder), tf.int32))
# grads tensor:
grads = average_gradients(tower_grads) # average gradient over all GPUs
# apply gradients operation:
with tf.control_dependencies([tf.group(*tf.get_collection(tf.GraphKeys.UPDATE_OPS))]):
train_op = opt.apply_gradients(grads, global_step=global_step)
# input operations:
tr_videos_op, _, tr_actor_labels_op = create_videos_reading_ops(is_train=True, is_val=False, cfg=cfg)
val_videos_op, _, val_actor_labels_op = create_videos_reading_ops(is_train=False, is_val=True, cfg=cfg)
test_videos_op, _, test_actor_labels_op = create_videos_reading_ops(is_train=False, is_val=False, cfg=cfg)
# initialize operations:
init_op = tf.group(tf.local_variables_initializer(), tf.global_variables_initializer())
return (graph, init_op, train_op,
logits_op, acc_op, correct_count_op, loss_op,
tr_videos_op, tr_actor_labels_op, val_videos_op, val_actor_labels_op, test_videos_op, test_actor_labels_op,
videos_placeholder, labels_placeholder, istraining_placeholder,
varlist_budget, varlist_degrad)