def graph(x, y, i, x_max, x_min, grad):
eps = 2.0 * FLAGS.max_epsilon / 255.0
num_iter = FLAGS.num_iter
alpha = eps / num_iter
momentum = FLAGS.momentum
num_classes = 1001
# should keep original x here for output
with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
logits_v3, end_points_v3 = inception_v3.inception_v3(
input_diversity(x), num_classes=num_classes, is_training=False)
logits_v3_rotated, _ = inception_v3.inception_v3(
rotate(x), num_classes=num_classes, is_training=False, reuse=True)
with slim.arg_scope(inception_v4.inception_v4_arg_scope()):
logits_v4, end_points_v4 = inception_v4.inception_v4(
input_diversity(x), num_classes=num_classes, is_training=False)
logits_v4_rotated, _ = inception_v4.inception_v4(
rotate(x), num_classes=num_classes, is_training=False, reuse=True)
with slim.arg_scope(inception_resnet_v2.inception_resnet_v2_arg_scope()):
logits_res_v2, end_points_res_v2 = inception_resnet_v2.inception_resnet_v2(
input_diversity(x),
num_classes=num_classes,
is_training=False,
reuse=True)
logits_res_v2_rotated, _ = inception_resnet_v2.inception_resnet_v2(
rotate(x), num_classes=num_classes, is_training=False, reuse=True)
with slim.arg_scope(resnet_v2.resnet_arg_scope()):
logits_resnet, end_points_resnet = resnet_v2.resnet_v2_152(
input_diversity(x), num_classes=num_classes, is_training=False)
logits_resnet_rotated, _ = resnet_v2.resnet_v2_152(
rotate(x), num_classes=num_classes, is_training=False, reuse=True)
logits = (logits_v3 + logits_v4 + logits_res_v2 + logits_resnet +
logits_v3_rotated + logits_v4_rotated + logits_res_v2_rotated +
logits_resnet_rotated) / 8
auxlogits = (end_points_v3['AuxLogits'] + end_points_v4['AuxLogits'] +
end_points_res_v2['AuxLogits']) / 3
cross_entropy = tf.losses.softmax_cross_entropy(y,
logits,
label_smoothing=0.0,
weights=1.0)
cross_entropy += tf.losses.softmax_cross_entropy(y,
auxlogits,
label_smoothing=0.0,
weights=0.4)
noise = tf.gradients(cross_entropy, x)[0]
# noise = tf.nn.depthwise_conv2d(noise, stack_kernel, strides=[1, 1, 1, 1], padding='SAME')
noise = noise / tf.reduce_mean(tf.abs(noise), [1, 2, 3], keep_dims=True)
noise = momentum * grad + noise
x = x + alpha * tf.sign(noise)
x = tf.clip_by_value(x, x_min, x_max)
i = tf.add(i, 1)
return x, y, i, x_max, x_min, noise
def __init__(self, source_image_size, use_smoothed_grad=False):
self.image_size = 299
self.source_image_size = source_image_size
self.num_classes = 1001
self.predictions_is_correct = False
batch_shape = [None, self.image_size, self.image_size, 3]
self.x_input = tf.placeholder(tf.float32, shape=batch_shape)
self.target_label = tf.placeholder(tf.int32, shape=[None])
target_onehot = tf.one_hot(self.target_label, self.num_classes)
with slim.arg_scope(resnet_v2.resnet_arg_scope()):
logits, end_points = resnet_v2.resnet_v2_152(
self.x_input, num_classes=self.num_classes, is_training=False)
self.predicted_labels = tf.argmax(end_points['predictions'], 1)
#logits -= tf.reduce_min(logits)
#real = tf.reduce_max(logits * target_onehot, 1)
#other = tf.reduce_max(logits * (1 - target_onehot), 1)
#self.loss = other - real
self.loss = tf.nn.softmax_cross_entropy_with_logits(
labels=target_onehot, logits=logits)
self.grad = 2 * tf.gradients(self.loss, self.x_input)[0]
if use_smoothed_grad:
self.grad = tf.nn.depthwise_conv2d(self.grad,
stack_kernel,
strides=[1, 1, 1, 1],
padding='SAME')
saver = tf.train.Saver(slim.get_model_variables(scope='resnet_v2'))
self.sess = tf.get_default_session()
saver.restore(self.sess, 'resnet_v2_152.ckpt')
def test_resnet_v2_152(img_dir):
"""
Test ResNet-V1-152 with a single image.
:param img_dir: Path of the image to be classified
:return: classification result and probability of a single image
"""
img = cv2.imread(img_dir)
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
img = cv2.resize(img, (224, 224)) / 255
img = img.reshape((1, 224, 224, 3))
tf.reset_default_graph()
inputs = tf.placeholder(name='input_images',
shape=[None, 224, 224, 3],
dtype=tf.float32)
with slim.arg_scope(resnet_arg_scope()):
_, _ = resnet_v2_152(inputs, 1001, is_training=False)
with tf.Session() as sess:
tf.train.Saver().restore(sess, './models/resnet_v2_152.ckpt')
inputs = sess.graph.get_tensor_by_name('input_images:0')
outputs = sess.graph.get_tensor_by_name(
'resnet_v2_152/SpatialSqueeze:0')
pred = tf.argmax(tf.nn.softmax(outputs), axis=1)[0]
prob = tf.reduce_max(tf.nn.softmax(outputs), axis=1)[0]
pred, prob = sess.run([pred, prob], feed_dict={inputs: img})
name = label_dict[pred]
print('Result of ResNet-V1-152:', name, prob)
return name, prob
def inference_resnet_v2_152(x_input, dropout_keep_prob=1, num_classes=1001):
with slim.arg_scope(resnet_v2.resnet_arg_scope()):
logits, _ = resnet_v2.resnet_v2_152(x_input,
num_classes=num_classes, is_training=False)
probs = tf.nn.softmax(logits)
model_vars = [var for var in tf.global_variables() \
if var.name.startswith('resnet_v2_152/')]
return probs, logits, model_vars
def resnet_v2_152(inputs, is_training, opts):
with slim.arg_scope(resnet_v2.resnet_arg_scope(
weight_decay=opts.weight_decay,
batch_norm_decay=opts.batch_norm_decay,
batch_norm_epsilon=opts.batch_norm_epsilon,
activation_fn=tf.nn.relu)):
return resnet_v2.resnet_v2_152(
inputs,
num_classes=opts.num_classes,
is_training=is_training,
global_pool=opts.global_pool,
output_stride=None,
spatial_squeeze=opts.spatial_squeeze,
reuse=None)
def create(self, images, num_classes, is_training):
"""See baseclass."""
with slim.arg_scope(resnet_v2.resnet_arg_scope()):
_, endpoints = resnet_v2.resnet_v2_152(
images, num_classes, is_training=is_training, spatial_squeeze=False)
# Resnet's "predictions" endpoint is (n, 1, 1, m) but we really
# want to have an (n, m) "Predictions" endpoint. We add a squeeze
# op here to make that happen.
endpoints['Predictions'] = tf.squeeze(
endpoints['predictions'], [1, 2], name='SqueezePredictions')
# Likewise, the endpoint "resnet_v2_152/logits" should be squeezed to
# "Logits"
endpoints['Logits'] = tf.squeeze(
endpoints['resnet_v2_152/logits'], [1, 2], name='SqueezeLogits')
return endpoints
def __call__(self, x_input, batch_size=None, is_training=False):
"""Constructs model and return probabilities for given input."""
reuse = True if self.built else None
with slim.arg_scope(resnet_utils.resnet_arg_scope()):
with tf.variable_scope(self.ckpt):
logits, end_points = resnet_v2.resnet_v2_152(
x_input, num_classes=self.num_classes, is_training=is_training,
reuse=reuse)
preds = tf.argmax(logits, axis=1)
self.built = True
self.logits = logits
self.preds = preds
return logits
def create(self, images, num_classes, is_training):
"""See baseclass."""
with slim.arg_scope(resnet_v2.resnet_arg_scope()):
_, endpoints = resnet_v2.resnet_v2_152(
images, num_classes, is_training=is_training, spatial_squeeze=False)
# Resnet's "predictions" endpoint is (n, 1, 1, m) but we really
# want to have an (n, m) "Predictions" endpoint. We add a squeeze
# op here to make that happen.
endpoints['Predictions'] = tf.squeeze(
endpoints['predictions'], [1, 2], name='SqueezePredictions')
# Likewise, the endpoint "resnet_v2_152/logits" should be squeezed to
# "Logits"
endpoints['Logits'] = tf.squeeze(
endpoints['resnet_v2_152/logits'], [1, 2], name='SqueezeLogits')
return endpoints
def graph(x, adv, y, t_y, i, x_max, x_min, grad, amplification):
target_one_hot = tf.one_hot(t_y, 1001)
true_one_hot = tf.one_hot(y, 1001)
eps = 2.0 * FLAGS.max_epsilon / 255.0
num_iter = FLAGS.num_iter
alpha = eps / num_iter
alpha_beta = alpha
gamma = alpha_beta
momentum = FLAGS.momentum
num_classes = 1001
# with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
# logits_v3, end_points_v3 = inception_v3.inception_v3(
# adv, num_classes=num_classes, is_training=False, reuse = True)
# auxlogit_v3 = end_points_v3['AuxLogits']
with slim.arg_scope(inception_v4.inception_v4_arg_scope()):
logits_v4, end_points_v4 = inception_v4.inception_v4(
adv, num_classes=num_classes, is_training=False, reuse=True)
auxlogit_v4 = end_points_v4['AuxLogits']
with slim.arg_scope(resnet_v2.resnet_arg_scope()):
logits_resnet_152, end_points_resnet = resnet_v2.resnet_v2_152(
adv, num_classes=num_classes, is_training=False, reuse=True)
with slim.arg_scope(inception_resnet_v2.inception_resnet_v2_arg_scope()):
logits_Incres, end_points_IR = inception_resnet_v2.inception_resnet_v2(
adv, num_classes=num_classes, is_training=False, reuse=True)
auxlogit_IR = end_points_IR['AuxLogits']
logits = (logits_Incres + logits_resnet_152 + logits_v4) / 3.0
auxlogit = (auxlogit_IR + auxlogit_v4) / 2.0
target_cross_entropy = tf.losses.softmax_cross_entropy(target_one_hot,
logits,
label_smoothing=0.0,
weights=1.0)
target_cross_entropy += tf.losses.softmax_cross_entropy(
target_one_hot, auxlogit, label_smoothing=0.0, weights=1.0)
noise = tf.gradients(target_cross_entropy, adv)[0]
adv = adv - alpha * n_staircase_sign(noise, num_of_K)
adv = tf.clip_by_value(adv, x_min, x_max)
i = tf.add(i, 1)
return x, adv, y, t_y, i, x_max, x_min, noise, amplification
def target_model(x):
num_classes=1001
with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
logits_v3, end_points_v3 = inception_v3.inception_v3(
input_diversity(x), num_classes=num_classes, is_training=False)
with slim.arg_scope(inception_v4.inception_v4_arg_scope()):
logits_v4, end_points_v4 = inception_v4.inception_v4(
input_diversity(x), num_classes=num_classes, is_training=False)
with slim.arg_scope(inception_resnet_v2.inception_resnet_v2_arg_scope()):
logits_res_v2, end_points_res_v2 = inception_resnet_v2.inception_resnet_v2(
input_diversity(x), num_classes=num_classes, is_training=False, reuse=True)
with slim.arg_scope(resnet_v2.resnet_arg_scope()):
logits_resnet, end_points_resnet = resnet_v2.resnet_v2_152(
input_diversity(x), num_classes=num_classes, is_training=False)
with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
logits_adv_v3, end_points_adv_v3 = inception_v3.inception_v3(
input_diversity(x), num_classes=num_classes, is_training=False, scope='AdvInceptionV3')
with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
logits_ens3_adv_v3, end_points_ens3_adv_v3 = inception_v3.inception_v3(
input_diversity(x), num_classes=num_classes, is_training=False, scope='Ens3AdvInceptionV3')
with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
logits_ens4_adv_v3, end_points_ens4_adv_v3 = inception_v3.inception_v3(
input_diversity(x), num_classes=num_classes, is_training=False, scope='Ens4AdvInceptionV3')
with slim.arg_scope(inception_resnet_v2.inception_resnet_v2_arg_scope()):
logits_ensadv_res_v2, end_points_ensadv_res_v2 = inception_resnet_v2.inception_resnet_v2(
input_diversity(x), num_classes=num_classes, is_training=False, scope='EnsAdvInceptionResnetV2')
logits = (logits_v3 +1.6*logits_adv_v3+logits_v4 +logits_res_v2 + logits_resnet+logits_ens3_adv_v3+logits_ens4_adv_v3+logits_ensadv_res_v2) / 8.6
auxlogits = ( 1.6*end_points_adv_v3['AuxLogits'] +end_points_v3['AuxLogits']+end_points_ens3_adv_v3['AuxLogits']
+ end_points_v4['AuxLogits']+end_points_res_v2['AuxLogits']+end_points_ens4_adv_v3['AuxLogits']+end_points_ensadv_res_v2['AuxLogits']) / 7.6
return logits,auxlogits
def model_fn(features,labels,mode,params):
global_step = tf.train.get_global_step()
inputs = features['MRI']
logits, end_points= resnet_v2.resnet_v2_152(inputs=inputs,
num_classes=30)
# predict mode
predicted_classes = tf.argmax(logits, 1)
if mode == tf.estimator.ModeKeys.PREDICT:
predictions = {
'class':predicted_classes,
'prob':tf.nn.softmax(logits)
}
return tf.estimator.EstimatorSpec(mode,predictions=predictions)
with tf.name_scope('accuracy'):
accuracy = tf.metrics.accuracy(labels=labels,predictions=predicted_classes)
my_acc = tf.reduce_mean(tf.cast(tf.equal(labels, predicted_classes), tf.float32))
tf.summary.scalar('accuracy',my_acc)
# compute loss
loss = tf.losses.sparse_softmax_cross_entropy(labels=labels,logits=logits)
# hook
train_hook_list = []
train_tensors_log = {'accuracy':accuracy[1],
'my_acc': my_acc,
'loss':loss,
'global_step':global_step}
train_hook_list.append(tf.train.LoggingTensorHook(tensors=train_tensors_log,every_n_iter=200))
# training op
if mode == tf.estimator.ModeKeys.TRAIN:
optimizer = tf.train.MomentumOptimizer(learning_rate=0.001,momentum=0.9)
train_op = optimizer.minimize(loss,global_step=tf.train.get_global_step())
return tf.estimator.EstimatorSpec(mode=mode,loss=loss,train_op=train_op,training_hooks=train_hook_list)
# compute evaluation metrics
eval_metric_ops = {
'accuracy':tf.metrics.accuracy(labels=labels,predictions=predicted_classes)
}
return tf.estimator.EstimatorSpec(mode=mode,loss=loss,eval_metric_ops=eval_metric_ops)
def __init__(self):
batch_shape = [None, 299, 299, 3]
self.x_input = tf.placeholder(tf.float32, shape=batch_shape)
self.target_label = tf.placeholder(tf.int32, shape=[None])
target_onehot = tf.one_hot(self.target_label, 1001)
with slim.arg_scope(resnet_v2.resnet_arg_scope()):
logits, end_points = resnet_v2.resnet_v2_152(self.x_input,
num_classes=1001,
is_training=False)
self.predicted_labels = tf.argmax(end_points['predictions'], 1)
#logits -= tf.reduce_min(logits)
#real = tf.reduce_max(logits * target_onehot, 1)
#other = tf.reduce_max(logits * (1 - target_onehot), 1)
#self.loss = other - real
self.loss = tf.nn.softmax_cross_entropy_with_logits(
labels=target_onehot, logits=logits)
self.grad = 2 * tf.gradients(self.loss, self.x_input)[0]
saver = tf.train.Saver(slim.get_model_variables(scope='resnet_v2'))
self.sess = tf.get_default_session()
saver.restore(self.sess, 'resnet_v2_152.ckpt')
def train():
eps=2.0*float(FLAGS.max_epsilon)/256.0;
tf.logging.set_verbosity(tf.logging.INFO);
with tf.Graph().as_default():
# Design architecture
# input
x_data = tf.placeholder(tf.float32, [None, FLAGS.img_height, FLAGS.img_width,3], name="x_data")
y_label = tf.placeholder(tf.float32, [None, FLAGS.num_classes], name="y_label")
# generator
x_generated, g_params = build_generator(x_data,FLAGS);
x_generated = x_generated * eps;
x_generated = x_data + x_generated;
# discriminator(inception v3)
with slim.arg_scope(inception.inception_v3_arg_scope()):
_, end_points = inception.inception_v3(
x_generated, num_classes=FLAGS.num_classes, is_training=False)
predicted_labels = tf.argmax(end_points['Predictions'], 1);
predicted_logits = end_points['Logits'];
disc_var_list=slim.get_model_variables();
# discriminator(resnet v2 50)
x_generated2=tf.image.resize_bilinear(x_generated,[224,224],align_corners=False);
with slim.arg_scope(resnet_utils.resnet_arg_scope()):
_, end_points2 = resnet_v2.resnet_v2_50(
x_generated2, num_classes=FLAGS.num_classes, is_training=False)
predicted_labels2 = tf.argmax(end_points2['predictions'], 1);
predicted_logits2 = end_points2['predictions'];
disc_var_list2=slim.get_model_variables()[len(disc_var_list):];
# discriminator(resnet v2 152)
with slim.arg_scope(resnet_utils.resnet_arg_scope()):
_, end_points3 = resnet_v2.resnet_v2_152(
x_generated2, num_classes=FLAGS.num_classes, is_training=False)
predicted_labels3 = tf.argmax(end_points3['predictions'], 1);
predicted_logits3 = end_points3['predictions'];
disc_var_list3=slim.get_model_variables()[(len(disc_var_list)+len(disc_var_list2)):];
# average
predicted_prob_avg = (end_points['Predictions']+end_points2['predictions']+end_points3['predictions'])/5.0;
predicted_labels_avg = tf.argmax((end_points['Predictions']+end_points2['predictions']+end_points3['predictions'])/5.0, 1);
# loss and optimizer
gen_acc=tf.reduce_mean(tf.cast(tf.equal(predicted_labels,tf.argmax(y_label,1)),tf.float32));
cross_entropy=tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=predicted_logits, labels=y_label));
gen_acc2=tf.reduce_mean(tf.cast(tf.equal(predicted_labels2,tf.argmax(y_label,1)),tf.float32));
cross_entropy2=tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=predicted_logits2, labels=y_label));
gen_acc3=tf.reduce_mean(tf.cast(tf.equal(predicted_labels3,tf.argmax(y_label,1)),tf.float32));
cross_entropy3=tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=predicted_logits3, labels=y_label));
gen_acc_avg=tf.reduce_mean(tf.cast(tf.equal(predicted_labels_avg,tf.argmax(y_label,1)),tf.float32));
cross_entropy_avg=tf.reduce_mean(-tf.reduce_sum(y_label*tf.log(predicted_prob_avg),1));
infi_norm=tf.reduce_mean(tf.norm(tf.reshape(abs(x_data-x_generated),[-1,FLAGS.img_size]),ord=np.inf,axis=1));
g_loss=-1*cross_entropy_avg;
optimizer = tf.train.AdamOptimizer(0.0001)
g_trainer = optimizer.minimize(g_loss, var_list=g_params)
# get the data and label
img_list=np.sort(glob.glob(FLAGS.input_folder+"*.png"));
total_data=np.zeros((len(img_list),FLAGS.img_height,FLAGS.img_width,3),dtype=float);
for i in range(len(img_list)):
total_data[i]=imread(img_list[i],mode='RGB').astype(np.float) / 255.0;
total_data[i]=total_data[i]*2.0-1.0; # 0~1 -> -1~1
val_data=np.copy(total_data[0]);
f=open(FLAGS.label_folder+"true_label","r");
total_label2=np.array([i[:-1].split(",")[1] for i in f.readlines()],dtype=int);
total_label=np.zeros((len(total_data),FLAGS.num_classes),dtype=int);
for i in range(len(total_data)):
total_label[i,total_label2[i]]=1;
val_label=np.copy(total_label[0]);
# shuffle
total_idx=range(len(total_data)); np.random.shuffle(total_idx);
total_data=total_data[total_idx];total_label=total_label[total_idx];
# Run computation
saver = tf.train.Saver(disc_var_list);
saver2 = tf.train.Saver(disc_var_list2);
saver3 = tf.train.Saver(disc_var_list3);
saver_gen = tf.train.Saver(g_params);
# initialization
init = tf.global_variables_initializer();
with tf.Session() as sess:
sess.run(init)
saver.restore(sess,FLAGS.checkpoint_path+FLAGS.checkpoint_file_name);
saver2.restore(sess,FLAGS.checkpoint_path+FLAGS.checkpoint_file_name2);
saver3.restore(sess,FLAGS.checkpoint_path+FLAGS.checkpoint_file_name3);
# training
for i in range(FLAGS.max_epoch):
tr_infi=0;
tr_ce=0;
tr_gen_acc=0;
tr_ce2=0;
tr_gen_acc2=0;
tr_ce3=0;
tr_gen_acc3=0;
tr_ce_avg=0;
tr_gen_acc_avg=0;
for j in range(len(total_data) / FLAGS.batch_size):
batch_data=total_data[j*FLAGS.batch_size:(j+1)*FLAGS.batch_size];
batch_label=total_label[j*FLAGS.batch_size:(j+1)*FLAGS.batch_size];
acc_p_a,ce_p_a,acc_p3,ce_p3,acc_p2,ce_p2,acc_p,ce_p,infi_p,_=sess.run([gen_acc_avg,cross_entropy_avg,gen_acc3,cross_entropy3,gen_acc2,cross_entropy2,gen_acc,cross_entropy,infi_norm,g_trainer],feed_dict={x_data: batch_data, y_label: batch_label});
tr_infi+=infi_p;
tr_ce+=ce_p;
tr_gen_acc+=acc_p;
tr_ce2+=ce_p2;
tr_gen_acc2+=acc_p2;
tr_ce3+=ce_p3;
tr_gen_acc3+=acc_p3;
tr_ce_avg+=ce_p_a;
tr_gen_acc_avg+=acc_p_a;
print(str(i+1)+" Epoch InfiNorm:"+str(tr_infi/(j+1))+",CE: "+str(tr_ce/(j+1))+",Acc: "+str(tr_gen_acc/(j+1))+",CE2: "+str(tr_ce2/(j+1))+",Acc2: "+str(tr_gen_acc2/(j+1))+",CE3: "+str(tr_ce3/(j+1))+",Acc3: "+str(tr_gen_acc3/(j+1))+",Acc_avg: "+str(tr_gen_acc_avg/(j+1))+",CE_avg: "+str(tr_ce_avg/(j+1)));
total_idx=range(len(total_data)); np.random.shuffle(total_idx);
total_data=total_data[total_idx];total_label=total_label[total_idx];
saver_gen.save(sess,"my-models_iv3_rv250_rv2152_avg2/my-model_"+str(FLAGS.max_epsilon)+".ckpt");
def upsampled_ResNet(image_batch, num_classes, is_training):
"""Upsampled ResNet V2 152 architecture as introduced by Chen et al. in "DeepLab:Semantic Image Segmentation with
Deep Convolutional Nets, Atrous Convolution, andFully Connected CRFs". The ResNet V2 152 network downsamples an
input by afactor specifyable by the user, then bilinear interpolation is used to upsample theprediction to the size
of the original input.
Parameters
----------
image_batch : tensor
tensor of shape [batch_size, height, width, depth] containing the image batch on which inference should be
performed
num_classes : int
specifies the number of classes to predict
is_training : boolean
argument defining if the network is trained or tested. required by the ResNet network definition. must always
be set to false to run ResNet in segmentation mode!
Returns
----------
upsampled_logits : tensor
tensor of shape [batch_size, height, width, num_classes] containing the upsampled unscaled log probabilities
from resnet_V2_152
up_resnet_var_mapping : dict {string: variable}
dictionary to map variables from ResNet V2 152 name scope to upsampled ResNet name scope
"""
image_batch_float = tf.to_float(image_batch)
with tf.variable_scope("up_ResNet_vars") as up_resnet_var_scope:
with slim.arg_scope(resnet_v2.resnet_arg_scope()):
# resnet model definition
resnet_logits, end_points = resnet_v2.resnet_v2_152(image_batch_float,
num_classes,
is_training=is_training,
global_pool=False,
spatial_squeeze=False,
output_stride=8)
resnet_logits_shape = tf.shape(resnet_logits)
# upsampling filters
with tf.variable_scope("Upsampling_filt_8"):
upsampling_filter_8_val = bilinear_upsampling_weights(8, num_classes)
upsampling_filter_8 = tf.constant(upsampling_filter_8_val)
# perform upsampling
with tf.variable_scope("8xUpsampling"):
upsampled_logits_shape = tf.stack([resnet_logits_shape[0],
resnet_logits_shape[1] * 8,
resnet_logits_shape[2] * 8,
resnet_logits_shape[3]])
upsampled_logits = tf.nn.conv2d_transpose(resnet_logits, upsampling_filter_8,
upsampled_logits_shape, [1, 8, 8, 1], name="Upsampling_8")
# create a variable mapping from resnet namescope to upsampled resnet namescope
up_resnet_var_mapping = {}
up_resnet_vars = slim.get_variables(up_resnet_var_scope)
for var in up_resnet_vars:
resnet_vars = var.name[len(up_resnet_var_scope.name)+1:-2]
up_resnet_var_mapping[resnet_vars] = var
return upsampled_logits, up_resnet_var_mapping
def target_graph(x, y, i, x_max, x_min, grad, label, x_ori):
eps = 2.0 * max_epsilon / 255.0
alpha = eps / num_iter
num_classes = FLAGS.num_classes
# find adversarial example in perlin noise direction
x = x + tf.random_uniform(tf.shape(x), minval=-1e-2, maxval=1e-2) * \
create_perlin_noise(seed=None, color=True, batch_size=FLAGS.batch_size, image_size=FLAGS.image_height, normalize=True, precalc_fade=None)
# dropout
x = generate_jitter_sample(x_ori, x, fade_eps=0, width=np.random.randint(low=1, high=12))
image = (((x + 1.0) * 0.5) * 255.0)
processed_imgs_in_v1 = preprocess_for_model(image, 'inception_v1')
with slim.arg_scope(inception.inception_v1_arg_scope()):
logits_inc_v1, end_points_inc_v1 = inception.inception_v1(
structure(processed_imgs_in_v1), num_classes=num_classes, is_training=False, scope='InceptionV1')
# rescale pixle range from [-1, 1] to [0, 255] for resnet_v1 and vgg's input
processed_imgs_res_v1_50 = preprocess_for_model(image, 'resnet_v1_50')
with slim.arg_scope(resnet_v1.resnet_arg_scope()):
logits_res_v1_50, end_points_res_v1_50 = resnet_v1.resnet_v1_50(
structure(processed_imgs_res_v1_50), num_classes=num_classes, is_training=False, scope='resnet_v1_50')
end_points_res_v1_50['logits'] = tf.squeeze(end_points_res_v1_50['while/resnet_v1_50/logits'], [1, 2])
end_points_res_v1_50['probs'] = tf.nn.softmax(end_points_res_v1_50['logits'])
# image = (((x + 1.0) * 0.5) * 255.0)#.astype(np.uint8)
processed_imgs_vgg_16 = preprocess_for_model(image, 'vgg_16')
with slim.arg_scope(vgg.vgg_arg_scope()):
logits_vgg_16, end_points_vgg_16 = vgg.vgg_16(
structure(processed_imgs_vgg_16), num_classes=num_classes, is_training=False, scope='vgg_16')
end_points_vgg_16['logits'] = end_points_vgg_16['vgg_16/fc8']
end_points_vgg_16['probs'] = tf.nn.softmax(end_points_vgg_16['logits'])
processed_imgs_in_v3 = preprocess_for_model(image, 'inception_v3')
with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
logits_in_v3, end_points_in_v3 = inception_v3.inception_v3(
structure_res(processed_imgs_in_v3), num_classes=num_classes, is_training=False, scope='InceptionV3')
processed_res_v2 = preprocess_for_model(image, 'resnet_v2_152')
with slim.arg_scope(resnet_v2.resnet_arg_scope()):
logits_res_v2, end_points_res_v2 = resnet_v2.resnet_v2_152(
structure_res(processed_res_v2), num_classes=num_classes, is_training=False, scope='resnet_v2_152')
end_points_res_v2['logits'] = tf.squeeze(end_points_res_v2['resnet_v2_152/logits'], [1, 2])
end_points_res_v2['probs'] = tf.nn.softmax(end_points_res_v2['logits'])
one_hot = tf.one_hot(y, num_classes)
# separate the loss
cross_entropy_v1 = tf.losses.softmax_cross_entropy(one_hot,
end_points_inc_v1['Logits'],
label_smoothing=0.0,
weights=1.0)
cross_entropy_re = tf.losses.softmax_cross_entropy(one_hot,
end_points_res_v1_50['logits'],
label_smoothing=0.0,
weights=1.0)
cross_entropy_vgg = tf.losses.softmax_cross_entropy(one_hot,
end_points_vgg_16['logits'],
label_smoothing=0.0,
weights=1.0)
cross_entropy_re2 = tf.losses.softmax_cross_entropy(one_hot,
end_points_res_v2['logits'],
label_smoothing=0.0,
weights=1.0)
cross_entropy_v3 = tf.losses.softmax_cross_entropy(one_hot,
end_points_in_v3['Logits'],
label_smoothing=0.0,
weights=1.0)
pred = tf.argmax(end_points_inc_v1['Predictions'] + end_points_res_v1_50['probs'] + end_points_vgg_16['probs']+end_points_res_v2['probs'], 1)
first_round = tf.cast(tf.equal(i, 0), tf.int64)
label = first_round * pred + (1 - first_round) * label
noise_re = tf.gradients(cross_entropy_re,x)[0]
noise_re2 = tf.gradients(cross_entropy_re2,x)[0]
noise_v1 = tf.gradients(cross_entropy_v1,x)[0]
noise_vgg = tf.gradients(cross_entropy_vgg,x)[0]
noise_v3 = tf.gradients(cross_entropy_v3,x)[0]
noise_re = tf.Print(noise_re, [i, cross_entropy_re, cross_entropy_re2, cross_entropy_v1, cross_entropy_vgg, cross_entropy_v3])
noise_re = noise_re / tf.reshape(
tf.contrib.keras.backend.std(tf.reshape(noise_re, [FLAGS.batch_size, -1]), axis=1),
[FLAGS.batch_size, 1, 1, 1])
noise_re2 = noise_re2 / tf.reshape(
tf.contrib.keras.backend.std(tf.reshape(noise_re2, [FLAGS.batch_size, -1]), axis=1),
[FLAGS.batch_size, 1, 1, 1])
noise_v1 = noise_v1 / tf.reshape(
tf.contrib.keras.backend.std(tf.reshape(noise_v1, [FLAGS.batch_size, -1]), axis=1),
[FLAGS.batch_size, 1, 1, 1])
noise_vgg = noise_vgg / tf.reshape(
tf.contrib.keras.backend.std(tf.reshape(noise_vgg, [FLAGS.batch_size, -1]), axis=1),
[FLAGS.batch_size, 1, 1, 1])
noise_v3 = noise_v3 / tf.reshape(
tf.contrib.keras.backend.std(tf.reshape(noise_v3, [FLAGS.batch_size, -1]), axis=1),
[FLAGS.batch_size, 1, 1, 1])
noise = momentum * grad + noise_re + noise_re2 + noise_v1 + noise_vgg + noise_v3
noise = noise / tf.reshape(
tf.contrib.keras.backend.std(tf.reshape(noise, [FLAGS.batch_size, -1]), axis=1),
[FLAGS.batch_size, 1, 1, 1])
x = x - alpha * tf.clip_by_value(tf.round(noise), -2, 2)
x = tf.clip_by_value(x, x_min, x_max)
i = tf.add(i, 1)
return x, y, i, x_max, x_min, noise, label, x_ori
def __init__(self, options):
num_classes = options.NUM_CLASSES
with tf.variable_scope("input"):
self.image_size = options.IMAGE_SIZE
self.x_input = tf.placeholder(
tf.float32, [None, self.image_size, self.image_size, 3],
name="x_input")
self.y_input = tf.placeholder(tf.float32, [None, num_classes],
name="y_input")
self.learning_rate = tf.placeholder(tf.float32,
name="learning_rate")
self.keep_prob = None
if options.PHASE == 'train':
if train_layers == 'default':
self.train_layers = self.DEFAULT_TRAIN_LAYERS
else:
self.train_layers = train_layers
# train
with arg_scope(
resnet_v2.resnet_arg_scope(
activation_fn=tf.nn.relu,
weight_decay=options.WEIGHT_DECAY)):
self.logits, _ = resnet_v2.resnet_v2_152(
self.x_input,
num_classes=num_classes,
is_training=True,
reuse=tf.AUTO_REUSE)
# validation
with arg_scope(
resnet_v2.resnet_arg_scope(
activation_fn=tf.nn.relu,
weight_decay=options.WEIGHT_DECAY)):
self.logits_val, _ = resnet_v2.resnet_v2_152(
self.x_input,
num_classes=num_classes,
is_training=False,
reuse=tf.AUTO_REUSE)
with tf.name_scope("loss"):
self.loss = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits_v2(
logits=self.logits, labels=self.y_input))
self.loss_val = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits_v2(
logits=self.logits_val, labels=self.y_input))
with tf.name_scope("train"):
self.global_step = tf.Variable(0,
name="global_step",
trainable=False)
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
var_list = [
v for v in tf.trainable_variables()
if v.name.split('/')[-2] in self.train_layers
or v.name.split('/')[-3] in self.train_layers
]
# var_list = tf.trainable_variables()
# var_list = [v for v in tf.trainable_variables() if
# v.name.split('/')[1] in self.train_layers]
gradients = tf.gradients(self.loss, var_list)
self.grads_and_vars = list(zip(gradients, var_list))
optimizer = get_optimizer(options.OPTIMIZER,
self.learning_rate)
with tf.control_dependencies(update_ops):
self.train_op = optimizer.apply_gradients(
grads_and_vars=self.grads_and_vars,
global_step=self.global_step)
else:
# Only Validation
with arg_scope(
resnet_v2.resnet_arg_scope(activation_fn=tf.nn.relu,
weight_decay=0.0)):
self.logits_val, _ = resnet_v2.resnet_v2_152(
self.x_input,
num_classes=num_classes,
is_training=False,
reuse=tf.AUTO_REUSE)
with tf.name_scope("probability"):
self.probability = tf.nn.softmax(self.logits_val,
name="probability")
with tf.name_scope("prediction"):
self.prediction = tf.argmax(self.logits_val, 1, name="prediction")
with tf.name_scope("accuracy"):
correct_prediction = tf.equal(self.prediction,
tf.argmax(self.y_input, 1))
self.accuracy = tf.reduce_mean(tf.cast(correct_prediction,
"float"),
name="accuracy")
def graph(x, y, i, x_max, x_min, grad, amplification):
one_hot = tf.one_hot(y, FLAGS.num_classes)
eps = 2.0 * FLAGS.max_epsilon / 255.0
num_iter = FLAGS.num_iter
alpha = eps / num_iter
momentum = FLAGS.momentum
# amplification factor
beta = alpha * FLAGS.amplification_factor
gamma = beta
# DIM: https://arxiv.org/abs/1803.06978
# input_diversity(FLAG, x)
with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
logits_v3, end_points_v3 = inception_v3.inception_v3(
x, num_classes=FLAGS.num_classes, is_training=False)
auxlogits_v3 = end_points_v3['AuxLogits']
with slim.arg_scope(inception_v4.inception_v4_arg_scope()):
logits_v4, end_points_v4 = inception_v4.inception_v4(
x, num_classes=FLAGS.num_classes, is_training=False)
auxlogits_v4 = end_points_v4['AuxLogits']
with slim.arg_scope(resnet_v2.resnet_arg_scope()):
logits_resnet, end_points_resnet = resnet_v2.resnet_v2_152(
x, num_classes=FLAGS.num_classes, is_training=False)
with slim.arg_scope(inception_resnet_v2.inception_resnet_v2_arg_scope()):
logits_Incres, end_points_IR = inception_resnet_v2.inception_resnet_v2(
x, num_classes=FLAGS.num_classes, is_training=False)
auxlogits_IR = end_points_IR['AuxLogits']
logits = (logits_v3 + logits_v4 + logits_resnet + logits_Incres) / 4.0
auxlogits = (auxlogits_v3 + auxlogits_v4 + auxlogits_IR) / 3.0
cross_entropy = tf.losses.softmax_cross_entropy(one_hot,
logits,
label_smoothing=0.0,
weights=1.0)
cross_entropy += tf.losses.softmax_cross_entropy(one_hot,
auxlogits,
label_smoothing=0.0,
weights=1.0)
noise = tf.gradients(cross_entropy, x)[0]
# TI-FGSM: https://arxiv.org/pdf/1904.02884.pdf
# noise = tf.nn.depthwise_conv2d(noise, T_kern, strides=[1, 1, 1, 1], padding='SAME')
# MI-FGSM: https://arxiv.org/pdf/1710.06081.pdf
# noise = noise / tf.reduce_mean(tf.abs(noise), [1, 2, 3], keep_dims=True)
# noise = momentum * grad + noise
# Project cut noise
amplification += beta * tf.sign(noise)
cut_noise = tf.clip_by_value(abs(amplification) - eps, 0.0,
10000.0) * tf.sign(amplification)
projection = gamma * tf.sign(project_noise(cut_noise, P_kern, kern_size))
# Occasionally, when the adversarial examples are crafted for an ensemble of networks with residual block by combined methods,
# you may neet to comment the following line to get better result.
amplification += projection
x = x + beta * tf.sign(noise) + projection
x = tf.clip_by_value(x, x_min, x_max)
i = tf.add(i, 1)
return x, y, i, x_max, x_min, noise, amplification
def graph(adv, y, t_y, i, x_max, x_min, grad, amplification):
target_one_hot = tf.one_hot(t_y, 1001)
eps = 2.0 * FLAGS.max_epsilon / 255.0
num_iter = FLAGS.num_iter
alpha = eps / num_iter
momentum = FLAGS.momentum
alpha_beta = alpha * FLAGS.amplification_factor
gamma = alpha_beta * FLAGS.project_factor
num_classes = 1001
# with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
# logits_v3, end_points_v3 = inception_v3.inception_v3(
# input_diversity(FLAGS, adv), num_classes=num_classes, is_training=False, reuse = True)
# # auxlogits_v3 = end_points_v3['AuxLogits']
with slim.arg_scope(inception_v4.inception_v4_arg_scope()):
logits_v4, end_points_v4 = inception_v4.inception_v4(
input_diversity(FLAGS, adv),
num_classes=num_classes,
is_training=False,
reuse=True)
# auxlogits_v4 = end_points_v4['AuxLogits']
with slim.arg_scope(resnet_v2.resnet_arg_scope()):
logits_resnet_152, end_points_resnet = resnet_v2.resnet_v2_152(
input_diversity(FLAGS, adv),
num_classes=num_classes,
is_training=False,
reuse=True)
with slim.arg_scope(resnet_v2.resnet_arg_scope()):
logits_resnet_101, end_points_resnet_101 = resnet_v2.resnet_v2_101(
input_diversity(FLAGS, adv),
num_classes=num_classes,
is_training=False,
reuse=True)
with slim.arg_scope(resnet_v2.resnet_arg_scope()):
logits_resnet_50, end_points_resnet_50 = resnet_v2.resnet_v2_50(
input_diversity(FLAGS, adv),
num_classes=num_classes,
is_training=False,
reuse=True)
with slim.arg_scope(inception_resnet_v2.inception_resnet_v2_arg_scope()):
logits_Incres, end_points_IR = inception_resnet_v2.inception_resnet_v2(
input_diversity(FLAGS, adv),
num_classes=num_classes,
is_training=False,
reuse=True)
# auxlogits_Incres = end_points_IR['AuxLogits']
logits = (logits_v4 + logits_resnet_152 + logits_resnet_101 +
logits_Incres + logits_resnet_50) / 5.0 / FLAGS.temperature
target_cross_entropy = tf.losses.softmax_cross_entropy(target_one_hot,
logits,
label_smoothing=0.0,
weights=1.0)
noise = tf.gradients(target_cross_entropy, adv)[0]
noise = tf.nn.depthwise_conv2d(noise,
T_kern,
strides=[1, 1, 1, 1],
padding='SAME')
# Project cut noise
amplification += alpha_beta * tf.sign(noise)
cut_noise = tf.clip_by_value(abs(amplification) - eps, 0.0,
10000.0) * tf.sign(amplification)
projection = gamma * tf.sign(project_noise(cut_noise, P_kern, kern_size))
adv = adv - alpha_beta * tf.sign(noise) - projection
adv = tf.clip_by_value(adv, x_min, x_max)
i = tf.add(i, 1)
return adv, y, t_y, i, x_max, x_min, noise, amplification
def main(_):
batch_shape = [FLAGS.batch_size, FLAGS.image_height, FLAGS.image_width, 3]
num_classes = 1001
# max_epsilon over checking
# get original images
origin_img_list=np.sort(glob.glob(FLAGS.origin_img_dir+"*.png"));
origin_imgs=np.zeros((len(origin_img_list),FLAGS.image_height,FLAGS.image_width,3),dtype=float);
for i in range(len(origin_img_list)):
origin_imgs[i]=imread(origin_img_list[i],mode='RGB').astype(np.float);
# get adv images
adv_img_list=np.sort(glob.glob(FLAGS.input_dir+"*.png"));
adv_imgs=np.zeros((len(adv_img_list),FLAGS.image_height,FLAGS.image_width,3),dtype=float);
for i in range(len(adv_img_list)):
adv_imgs[i]=imread(adv_img_list[i],mode='RGB').astype(np.float);
epsilon_list=np.linalg.norm(np.reshape(abs(origin_imgs-adv_imgs),[-1,FLAGS.image_height*FLAGS.image_width*3]),ord=np.inf,axis=1);
#print(epsilon_list);exit(1);
over_epsilon_list=np.zeros((len(origin_img_list),2),dtype=object);
cnt=0;
for i in range(len(origin_img_list)):
file_name=origin_img_list[i].split("/")[-1];
file_name=file_name.split(".")[0];
over_epsilon_list[i,0]=file_name;
if(epsilon_list[i]>FLAGS.max_epsilon):
over_epsilon_list[i,1]="1";
cnt+=1;
tf.logging.set_verbosity(tf.logging.INFO)
with tf.Graph().as_default():
# Prepare graph
x_input = tf.placeholder(tf.float32, shape=batch_shape)
if(FLAGS.checkpoint_file_name=="inception_v3.ckpt"):
with slim.arg_scope(inception.inception_v3_arg_scope()):
_, end_points = inception.inception_v3(
x_input, num_classes=num_classes, is_training=False)
predicted_labels = tf.argmax(end_points['Predictions'], 1)
elif(FLAGS.checkpoint_file_name=="inception_v4.ckpt"):
with slim.arg_scope(inception.inception_v4_arg_scope()):
_, end_points = inception.inception_v4(
x_input, num_classes=num_classes, is_training=False)
predicted_labels = tf.argmax(end_points['Predictions'], 1)
elif(FLAGS.checkpoint_file_name=="inception_resnet_v2_2016_08_30.ckpt"):
with slim.arg_scope(inception.inception_resnet_v2_arg_scope()):
_, end_points = inception.inception_resnet_v2(
x_input, num_classes=num_classes, is_training=False)
predicted_labels = tf.argmax(end_points['Predictions'], 1)
elif(FLAGS.checkpoint_file_name=="resnet_v2_101.ckpt"):
x_input2 = tf.image.resize_bilinear(x_input,[224,224],align_corners=False);
with slim.arg_scope(resnet_utils.resnet_arg_scope()):
_, end_points = resnet_v2.resnet_v2_101(
x_input2, num_classes=num_classes, is_training=False)
predicted_labels = tf.argmax(end_points['predictions'], 1)
elif(FLAGS.checkpoint_file_name=="resnet_v2_50.ckpt"):
x_input2 = tf.image.resize_bilinear(x_input,[224,224],align_corners=False);
with slim.arg_scope(resnet_utils.resnet_arg_scope()):
_, end_points = resnet_v2.resnet_v2_50(
x_input2, num_classes=num_classes, is_training=False)
predicted_labels = tf.argmax(end_points['predictions'], 1)
elif(FLAGS.checkpoint_file_name=="resnet_v2_152.ckpt"):
x_input2 = tf.image.resize_bilinear(x_input,[224,224],align_corners=False);
with slim.arg_scope(resnet_utils.resnet_arg_scope()):
_, end_points = resnet_v2.resnet_v2_152(
x_input2, num_classes=num_classes, is_training=False)
predicted_labels = tf.argmax(end_points['predictions'], 1)
elif(FLAGS.checkpoint_file_name=="inception_v1.ckpt"):
x_input2 = tf.image.resize_bilinear(x_input,[224,224],align_corners=False);
with slim.arg_scope(inception.inception_v1_arg_scope()):
_, end_points = inception.inception_v1(
x_input2, num_classes=num_classes, is_training=False)
predicted_labels = tf.argmax(end_points['Predictions'], 1)
elif(FLAGS.checkpoint_file_name=="inception_v2.ckpt"):
x_input2 = tf.image.resize_bilinear(x_input,[224,224],align_corners=False);
with slim.arg_scope(inception.inception_v2_arg_scope()):
_, end_points = inception.inception_v2(
x_input2, num_classes=num_classes, is_training=False)
predicted_labels = tf.argmax(end_points['Predictions'], 1)
# Resnet v1 and vgg are not working now
elif(FLAGS.checkpoint_file_name=="vgg_16.ckpt"):
x_input_list=tf.unstack(x_input,FLAGS.batch_size,0);
for i in range(FLAGS.batch_size):
x_input_list[i]=vgg_preprocessing.preprocess_image(x_input_list[i],224,224);
x_input2=tf.stack(x_input_list,0);
with slim.arg_scope(vgg.vgg_arg_scope()):
_, end_points = vgg.vgg_16(
x_input2, num_classes=num_classes-1, is_training=False)
predicted_labels = tf.argmax(end_points['vgg_16/fc8'], 1)+1
elif(FLAGS.checkpoint_file_name=="vgg_19.ckpt"):
x_input_list=tf.unstack(x_input,FLAGS.batch_size,0);
for i in range(FLAGS.batch_size):
x_input_list[i]=vgg_preprocessing.preprocess_image(x_input_list[i],224,224);
x_input2=tf.stack(x_input_list,0);
with slim.arg_scope(vgg.vgg_arg_scope()):
_, end_points = vgg.vgg_19(
x_input2, num_classes=num_classes-1, is_training=False)
predicted_labels = tf.argmax(end_points['vgg_19/fc8'], 1)+1
elif(FLAGS.checkpoint_file_name=="resnet_v1_50.ckpt"):
x_input_list=tf.unstack(x_input,FLAGS.batch_size,0);
for i in range(FLAGS.batch_size):
x_input_list[i]=vgg_preprocessing.preprocess_image(x_input_list[i],224,224);
x_input2=tf.stack(x_input_list,0);
with slim.arg_scope(resnet_utils.resnet_arg_scope()):
_, end_points = resnet_v1.resnet_v1_50(
x_input, num_classes=num_classes-1, is_training=False)
predicted_labels = tf.argmax(end_points['predictions'], 1)+1
elif(FLAGS.checkpoint_file_name=="resnet_v1_101.ckpt"):
x_input_list=tf.unstack(x_input,FLAGS.batch_size,0);
for i in range(FLAGS.batch_size):
x_input_list[i]=vgg_preprocessing.preprocess_image(x_input_list[i],224,224);
x_input2=tf.stack(x_input_list,0);
with slim.arg_scope(resnet_utils.resnet_arg_scope()):
_, end_points = resnet_v1.resnet_v1_101(
x_input2, num_classes=num_classes-1, is_training=False)
predicted_labels = tf.argmax(end_points['predictions'], 1)+1
elif(FLAGS.checkpoint_file_name=="resnet_v1_152.ckpt"):
x_input_list=tf.unstack(x_input,FLAGS.batch_size,0);
for i in range(FLAGS.batch_size):
x_input_list[i]=vgg_preprocessing.preprocess_image(x_input_list[i],224,224);
x_input2=tf.stack(x_input_list,0);
with slim.arg_scope(resnet_utils.resnet_arg_scope()):
_, end_points = resnet_v1.resnet_v1_152(
x_input2, num_classes=num_classes-1, is_training=False)
predicted_labels = tf.argmax(end_points['predictions'], 1)+1
# Run computation
saver = tf.train.Saver(slim.get_model_variables())
session_creator = tf.train.ChiefSessionCreator(
scaffold=tf.train.Scaffold(saver=saver),
checkpoint_filename_with_path=FLAGS.checkpoint_path+FLAGS.checkpoint_file_name,
master=FLAGS.master)
f=open(FLAGS.true_label,"r");
t_label_list=np.array([i[:-1].split(",") for i in f.readlines()]);
score=0;
with tf.train.MonitoredSession(session_creator=session_creator) as sess:
with tf.gfile.Open(FLAGS.output_file, 'w') as out_file:
for filenames, images in load_images(FLAGS.input_dir, batch_shape):
labels = sess.run(predicted_labels, feed_dict={x_input: images})
for filename, label in zip(filenames, labels):
f_name=filename.split(".")[0];
t_label=int(t_label_list[t_label_list[:,0]==f_name,1][0]);
if(t_label!=label):
if(over_epsilon_list[over_epsilon_list[:,0]==f_name,1]!="1"):
score+=1;
#out_file.write('{0},{1}\n'.format(filename, label))
print("Over max epsilon#: "+str(cnt));
print(str(FLAGS.max_epsilon)+" max epsilon Score: "+str(score));
# Pre-processing step.
image_preprocessing_fn = preprocessing_factory.get_preprocessing(
'simple', is_training=False)
image = image_preprocessing_fn(image, image_size, image_size)
images, labels, product_ids = tf.train.batch([image, label, product_id],
batch_size=batch_size,
num_threads=1,
capacity=5 * batch_size)
# Get the model
# network_fn = nets_factory.get_network_fn('resnet_v2_152', num_classes=num_classes, is_training=False)
with slim.arg_scope(resnet_v2.resnet_arg_scope(weight_decay=0.)):
logits, end_points = resnet_v2.resnet_v2_152(images,
num_classes=num_classes,
is_training=False)
#Obtain the trainable variables and a saver
variables_to_restore = slim.get_variables_to_restore()
saver = tf.train.Saver(variables_to_restore)
output_f = open(out_fn, 'w')
with tf.Session() as sess:
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord=coord)
sess.run(tf.global_variables_initializer())
saver.restore(sess, checkpoint_file)
num_iters = int(math.ceil(data_sizes[set_name] / float(batch_size)))
num_last_batch = batch_size - (
def Evaluator(x, y):
num_classes = 1001
# should keep original x here for output
with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
logits_v3, end_points_v3 = inception_v3.inception_v3(
x, num_classes=num_classes, is_training=False)
with slim.arg_scope(inception_v4.inception_v4_arg_scope()):
logits_v4, end_points_v4 = inception_v4.inception_v4(
x, num_classes=num_classes, is_training=False)
with slim.arg_scope(inception_resnet_v2.inception_resnet_v2_arg_scope()):
logits_res_v2, end_points_res_v2 = inception_resnet_v2.inception_resnet_v2(
x, num_classes=num_classes, is_training=False, reuse=True)
with slim.arg_scope(resnet_v2.resnet_arg_scope()):
logits_resnet, end_points_resnet = resnet_v2.resnet_v2_152(
x, num_classes=num_classes, is_training=False)
with slim.arg_scope(resnet_v2.resnet_arg_scope()):
logits_resnet_101, end_points_resnet_101 = resnet_v2.resnet_v2_101(
x, num_classes=num_classes, is_training=False, reuse=tf.AUTO_REUSE)
with slim.arg_scope(resnet_v2.resnet_arg_scope()):
logits_resnet_50, end_points_resnet_50 = resnet_v2.resnet_v2_50(
x, num_classes=num_classes, is_training=False, reuse=tf.AUTO_REUSE)
# with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
# logits_adv_v3, end_points_adv_v3 = inception_v3.inception_v3(
# x, num_classes=num_classes, is_training=False, scope='AdvInceptionV3')
#
# with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
# logits_ens3_adv_v3, end_points_ens3_adv_v3 = inception_v3.inception_v3(
# x, num_classes=num_classes, is_training=False, scope='Ens3AdvInceptionV3')
#
# with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
# logits_ens4_adv_v3, end_points_ens4_adv_v3 = inception_v3.inception_v3(
# x, num_classes=num_classes, is_training=False, scope='Ens4AdvInceptionV3')
#
# with slim.arg_scope(inception_resnet_v2.inception_resnet_v2_arg_scope()):
# logits_ensadv_res_v2, end_points_ensadv_res_v2 = inception_resnet_v2.inception_resnet_v2(
# x, num_classes=num_classes, is_training=False, scope='EnsAdvInceptionResnetV2')
# with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
# logits_ens_v3, end_points_ens_v3 = inception_v3.inception_v3(
# x, num_classes=num_classes, is_training=False)
# acc_v3 = tf.reduce_sum(tf.cast(tf.equal(tf.argmax(end_points_v3['Predictions'], 1), y), tf.float32))
# acc_v4 = tf.reduce_sum(tf.cast(tf.equal(tf.argmax(end_points_v4['Predictions'], 1), y), tf.float32))
# acc_res_v2 = tf.reduce_sum(tf.cast(tf.equal(tf.argmax(end_points_res_v2['Predictions'], 1), y), tf.float32))
# acc_resnet = tf.reduce_sum(tf.cast(tf.equal(tf.argmax(end_points_resnet['predictions'], 1), y), tf.float32))
# acc_resnet_50 = tf.reduce_sum(tf.cast(tf.equal(tf.argmax(end_points_resnet_50['predictions'], 1), y), tf.float32))
# acc_resnet_101 = tf.reduce_sum(tf.cast(tf.equal(tf.argmax(end_points_resnet_101['predictions'], 1), y), tf.float32))
# acc_ens_v3 = tf.reduce_sum(tf.cast(tf.equal(tf.argmax(end_points_ens_v3['predictions'], 1), y), tf.float32))
# top_k
top_k = FLAGS.top_k
acc_v3 = tf.reduce_sum(
tf.cast(tf.nn.in_top_k(end_points_v3['Predictions'], y, k=top_k),
tf.float32))
acc_v4 = tf.reduce_sum(
tf.cast(tf.nn.in_top_k(end_points_v4['Predictions'], y, k=top_k),
tf.float32))
acc_res_v2 = tf.reduce_sum(
tf.cast(tf.nn.in_top_k(end_points_res_v2['Predictions'], y, k=top_k),
tf.float32))
acc_resnet = tf.reduce_sum(
tf.cast(tf.nn.in_top_k(end_points_resnet['predictions'], y, k=top_k),
tf.float32))
acc_resnet_50 = tf.reduce_sum(
tf.cast(
tf.nn.in_top_k(end_points_resnet_50['predictions'], y, k=top_k),
tf.float32))
acc_resnet_101 = tf.reduce_sum(
tf.cast(
tf.nn.in_top_k(end_points_resnet_101['predictions'], y, k=top_k),
tf.float32))
# acc_adv_v3 = tf.reduce_sum(tf.cast(tf.nn.in_top_k(end_points_adv_v3['Predictions'], y, k=top_k), tf.float32))
# acc_ens3_adv_v3 = tf.reduce_sum(tf.cast(tf.nn.in_top_k(end_points_ens3_adv_v3['Predictions'], y, k=top_k), tf.float32))
# acc_ens4_adv_v3 = tf.reduce_sum(tf.cast(tf.nn.in_top_k(end_points_ens4_adv_v3['Predictions'], y, k=top_k), tf.float32))
# acc_ensadv_res_v2 = tf.reduce_sum(tf.cast(tf.nn.in_top_k(end_points_ensadv_res_v2['Predictions'], y, k=top_k), tf.float32))
# acc_ens_v3 = tf.reduce_sum(tf.cast(tf.equal(tf.argmax(end_points_ens_v3['predictions'], 1), y), tf.float32))
# pred1 = tf.argmax(end_points_v3['Predictions'], 1)
# pred2 = tf.argmax(end_points_v4['Predictions'], 1)
# pred3 = tf.argmax(end_points_res_v2['Predictions'], 1)
# pred4 = tf.argmax(end_points_resnet['predictions'], 1)
return acc_v3, acc_v4, acc_res_v2, acc_resnet, acc_resnet_50, acc_resnet_101, end_points_v3, end_points_v4, end_points_res_v2, end_points_resnet
def _construct_model(model_type='resnet_v1_50'):
"""Constructs model for the desired type of CNN.
Args:
model_type: Type of model to be used.
Returns:
end_points: A dictionary from components of the network to the corresponding
activations.
Raises:
ValueError: If the model_type is not supported.
"""
# Placeholder input.
images = array_ops.placeholder(dtypes.float32,
shape=(1, None, None, 3),
name=_INPUT_NODE)
# Construct model.
if model_type == 'inception_resnet_v2':
_, end_points = inception.inception_resnet_v2_base(images)
elif model_type == 'inception_resnet_v2-same':
_, end_points = inception.inception_resnet_v2_base(
images, align_feature_maps=True)
elif model_type == 'inception_v2':
_, end_points = inception.inception_v2_base(images)
elif model_type == 'inception_v2-no-separable-conv':
_, end_points = inception.inception_v2_base(images,
use_separable_conv=False)
elif model_type == 'inception_v3':
_, end_points = inception.inception_v3_base(images)
elif model_type == 'inception_v4':
_, end_points = inception.inception_v4_base(images)
elif model_type == 'alexnet_v2':
_, end_points = alexnet.alexnet_v2(images)
elif model_type == 'vgg_a':
_, end_points = vgg.vgg_a(images)
elif model_type == 'vgg_16':
_, end_points = vgg.vgg_16(images)
elif model_type == 'mobilenet_v1':
_, end_points = mobilenet_v1.mobilenet_v1_base(images)
elif model_type == 'mobilenet_v1_075':
_, end_points = mobilenet_v1.mobilenet_v1_base(images,
depth_multiplier=0.75)
elif model_type == 'resnet_v1_50':
_, end_points = resnet_v1.resnet_v1_50(images,
num_classes=None,
is_training=False,
global_pool=False)
elif model_type == 'resnet_v1_101':
_, end_points = resnet_v1.resnet_v1_101(images,
num_classes=None,
is_training=False,
global_pool=False)
elif model_type == 'resnet_v1_152':
_, end_points = resnet_v1.resnet_v1_152(images,
num_classes=None,
is_training=False,
global_pool=False)
elif model_type == 'resnet_v1_200':
_, end_points = resnet_v1.resnet_v1_200(images,
num_classes=None,
is_training=False,
global_pool=False)
elif model_type == 'resnet_v2_50':
_, end_points = resnet_v2.resnet_v2_50(images,
num_classes=None,
is_training=False,
global_pool=False)
elif model_type == 'resnet_v2_101':
_, end_points = resnet_v2.resnet_v2_101(images,
num_classes=None,
is_training=False,
global_pool=False)
elif model_type == 'resnet_v2_152':
_, end_points = resnet_v2.resnet_v2_152(images,
num_classes=None,
is_training=False,
global_pool=False)
elif model_type == 'resnet_v2_200':
_, end_points = resnet_v2.resnet_v2_200(images,
num_classes=None,
is_training=False,
global_pool=False)
else:
raise ValueError('Unsupported model_type %s.' % model_type)
return end_points
def _construct_model(model_type='resnet_v1_50'):
"""Constructs model for the desired type of CNN.
Args:
model_type: Type of model to be used.
Returns:
end_points: A dictionary from components of the network to the corresponding
activations.
Raises:
ValueError: If the model_type is not supported.
"""
# Placeholder input.
images = array_ops.placeholder(
dtypes.float32, shape=(1, None, None, 3), name=_INPUT_NODE)
# Construct model.
if model_type == 'inception_resnet_v2':
_, end_points = inception.inception_resnet_v2_base(images)
elif model_type == 'inception_resnet_v2-same':
_, end_points = inception.inception_resnet_v2_base(
images, align_feature_maps=True)
elif model_type == 'inception_v2':
_, end_points = inception.inception_v2_base(images)
elif model_type == 'inception_v2-no-separable-conv':
_, end_points = inception.inception_v2_base(
images, use_separable_conv=False)
elif model_type == 'inception_v3':
_, end_points = inception.inception_v3_base(images)
elif model_type == 'inception_v4':
_, end_points = inception.inception_v4_base(images)
elif model_type == 'alexnet_v2':
_, end_points = alexnet.alexnet_v2(images)
elif model_type == 'vgg_a':
_, end_points = vgg.vgg_a(images)
elif model_type == 'vgg_16':
_, end_points = vgg.vgg_16(images)
elif model_type == 'mobilenet_v1':
_, end_points = mobilenet_v1.mobilenet_v1_base(images)
elif model_type == 'mobilenet_v1_075':
_, end_points = mobilenet_v1.mobilenet_v1_base(
images, depth_multiplier=0.75)
elif model_type == 'resnet_v1_50':
_, end_points = resnet_v1.resnet_v1_50(
images, num_classes=None, is_training=False, global_pool=False)
elif model_type == 'resnet_v1_101':
_, end_points = resnet_v1.resnet_v1_101(
images, num_classes=None, is_training=False, global_pool=False)
elif model_type == 'resnet_v1_152':
_, end_points = resnet_v1.resnet_v1_152(
images, num_classes=None, is_training=False, global_pool=False)
elif model_type == 'resnet_v1_200':
_, end_points = resnet_v1.resnet_v1_200(
images, num_classes=None, is_training=False, global_pool=False)
elif model_type == 'resnet_v2_50':
_, end_points = resnet_v2.resnet_v2_50(
images, num_classes=None, is_training=False, global_pool=False)
elif model_type == 'resnet_v2_101':
_, end_points = resnet_v2.resnet_v2_101(
images, num_classes=None, is_training=False, global_pool=False)
elif model_type == 'resnet_v2_152':
_, end_points = resnet_v2.resnet_v2_152(
images, num_classes=None, is_training=False, global_pool=False)
elif model_type == 'resnet_v2_200':
_, end_points = resnet_v2.resnet_v2_200(
images, num_classes=None, is_training=False, global_pool=False)
else:
raise ValueError('Unsupported model_type %s.' % model_type)
return end_points
def train():
eps=2.0*float(FLAGS.max_epsilon)/256.0;
tf.logging.set_verbosity(tf.logging.INFO);
with tf.Graph().as_default():
# Design architecture
# input
is_training = tf.placeholder(tf.bool);
x_data = tf.placeholder(tf.float32, [None, FLAGS.img_height, FLAGS.img_width,3], name="x_data")
y_label = tf.placeholder(tf.float32, [None, FLAGS.num_classes], name="y_label")
y_label_ll = tf.placeholder(tf.float32, [None, FLAGS.num_classes], name="y_label_ll")
lam = tf.placeholder(tf.float32, [], name="lambda");
# generator
x_generated, g_params, bn_var_num = build_generator(x_data,is_training,FLAGS);
x_generated = x_generated * eps;
x_generated = x_data + x_generated;
# discriminator(inception v3)
with slim.arg_scope(inception.inception_v3_arg_scope()):
_, end_points = inception.inception_v3(
x_generated, num_classes=FLAGS.num_classes, is_training=False)
predicted_labels = tf.argmax(end_points['Predictions'], 1);
predicted_logits = end_points['Logits'];
disc_var_list=slim.get_model_variables()[bn_var_num:];
# discriminator(resnet v2 50)
x_generated2=tf.image.resize_bilinear(x_generated,[224,224],align_corners=False);
with slim.arg_scope(resnet_utils.resnet_arg_scope()):
_, end_points2 = resnet_v2.resnet_v2_50(
x_generated2, num_classes=FLAGS.num_classes, is_training=False)
predicted_labels2 = tf.argmax(end_points2['predictions'], 1);
predicted_logits2 = end_points2['predictions'];
disc_var_list2=slim.get_model_variables()[(bn_var_num+len(disc_var_list)):];
# discriminator(resnet v2 152)
with slim.arg_scope(resnet_utils.resnet_arg_scope()):
_, end_points3 = resnet_v2.resnet_v2_152(
x_generated2, num_classes=FLAGS.num_classes, is_training=False)
predicted_labels3 = tf.argmax(end_points3['predictions'], 1);
predicted_logits3 = end_points3['predictions'];
disc_var_list3=slim.get_model_variables()[(bn_var_num+len(disc_var_list)+len(disc_var_list2)):];
# discriminator(resnet v2 101)
with slim.arg_scope(resnet_utils.resnet_arg_scope()):
_, end_points4 = resnet_v2.resnet_v2_101(
x_generated2, num_classes=FLAGS.num_classes, is_training=False)
predicted_labels4 = tf.argmax(end_points4['predictions'], 1);
predicted_logits4 = end_points4['predictions'];
disc_var_list4=slim.get_model_variables()[(bn_var_num+len(disc_var_list)+len(disc_var_list2)+len(disc_var_list3)):];
# discriminator(inception v4)
with slim.arg_scope(inception.inception_v4_arg_scope()):
_, end_points5 = inception.inception_v4(
x_generated, num_classes=FLAGS.num_classes, is_training=False)
predicted_labels5 = tf.argmax(end_points5['Predictions'], 1);
predicted_logits5 = end_points['Logits'];
disc_var_list5=slim.get_model_variables()[(bn_var_num+len(disc_var_list)+len(disc_var_list2)+len(disc_var_list3)+len(disc_var_list4)):];
# average
predicted_prob_avg = (end_points['Predictions']+end_points2['predictions']+end_points3['predictions']+end_points4['predictions']+end_points5['Predictions'])/5.0;
predicted_labels_avg = tf.argmax((end_points['Predictions']+end_points2['predictions']+end_points3['predictions']+end_points4['predictions']+end_points5['Predictions'])/5.0, 1);
# loss and optimizer
gen_acc=tf.reduce_mean(tf.cast(tf.equal(predicted_labels_avg,tf.argmax(y_label,1)),tf.float32));
cross_entropy=tf.reduce_mean(-tf.reduce_sum(y_label*tf.log(predicted_prob_avg),1));
cross_entropy_ll=tf.reduce_mean(-tf.reduce_sum(y_label_ll*tf.log(predicted_prob_avg),1));
infi_norm=tf.reduce_mean(tf.norm(tf.reshape(abs(x_data-x_generated),[-1,FLAGS.img_size]),ord=np.inf,axis=1));
g_loss=-1*cross_entropy+cross_entropy_ll;
optimizer = tf.train.AdamOptimizer(0.0001)
g_trainer = optimizer.minimize(g_loss, var_list=g_params)
# get the data and label
img_list=np.sort(glob.glob(FLAGS.input_folder+"*.png"));
total_data=np.zeros((len(img_list),FLAGS.img_height,FLAGS.img_width,3),dtype=float);
for i in range(len(img_list)):
total_data[i]=imread(img_list[i],mode='RGB').astype(np.float) / 255.0;
total_data[i]=total_data[i]*2.0-1.0; # 0~1 -> -1~1
val_data=np.copy(total_data[0]);
f=open(FLAGS.label_folder+"true_label","r");
total_label2=np.array([i[:-1].split(",")[1] for i in f.readlines()],dtype=int);
total_label=np.zeros((len(total_data),FLAGS.num_classes),dtype=int);
for i in range(len(total_data)):
total_label[i,total_label2[i]]=1;
f=open("logits","r");
total_logits=np.array([i[:-1].split(",") for i in f.readlines()],dtype=float);
total_label_ll=np.zeros((len(total_data),FLAGS.num_classes),dtype=int);
for i in range(len(total_data)):
#total_logits[i,total_label2[i]]=0.0;
target_idx=np.argmin(total_logits[i]);
total_label_ll[i,target_idx]=1;
val_label=np.copy(total_label[0]);
# shuffle
total_idx=range(len(total_data)); np.random.shuffle(total_idx);
total_data=total_data[total_idx];total_label=total_label[total_idx];
# Run computation
saver = tf.train.Saver(disc_var_list);
saver2 = tf.train.Saver(disc_var_list2);
saver3 = tf.train.Saver(disc_var_list3);
saver4 = tf.train.Saver(disc_var_list4);
saver5 = tf.train.Saver(disc_var_list5);
saver_gen = tf.train.Saver(g_params);
"""
session_creator = tf.train.ChiefSessionCreator(
scaffold=tf.train.Scaffold(saver=saver),
checkpoint_filename_with_path=FLAGS.checkpoint_path,
master=FLAGS.master)
"""
# initialization
init = tf.global_variables_initializer();
with tf.Session() as sess:
#with tf.train.MonitoredSession() as sess:
sess.run(init)
saver.restore(sess,FLAGS.checkpoint_path+"inception_v3.ckpt");
saver2.restore(sess,FLAGS.checkpoint_path+"resnet_v2_50.ckpt");
saver3.restore(sess,FLAGS.checkpoint_path+"resnet_v2_152.ckpt");
saver4.restore(sess,FLAGS.checkpoint_path+"resnet_v2_101.ckpt");
saver5.restore(sess,FLAGS.checkpoint_path+"inception_v4.ckpt");
# tf board
tf.summary.scalar('reverse_cross_entropy',cross_entropy);
tf.summary.scalar('training_accuracy',gen_acc);
merged = tf.summary.merge_all()
train_writer = tf.summary.FileWriter('/tmp/nips17/attack_gan/learn_gan5_avg2_ll_'+str(FLAGS.max_epsilon), sess.graph)
# training
for i in range(FLAGS.max_epoch):
if(i>100):
lam_value=1.0;
else:
lam_value=1.0;
tr_ce=0;
tr_infi=0;
tr_gen_acc=0;
for j in range(len(total_data) / FLAGS.batch_size):
batch_data=total_data[j*FLAGS.batch_size:(j+1)*FLAGS.batch_size];
batch_label=total_label[j*FLAGS.batch_size:(j+1)*FLAGS.batch_size];
batch_label_ll=total_label_ll[j*FLAGS.batch_size:(j+1)*FLAGS.batch_size];
summary,tr_gen_acc_part,tr_ce_part,tr_infi_part,_=sess.run([merged,gen_acc,cross_entropy,infi_norm,g_trainer],feed_dict={is_training:True,x_data: batch_data, y_label: batch_label,y_label_ll:batch_label_ll,lam:lam_value});
tr_ce+=tr_ce_part;
tr_infi+=tr_infi_part;
tr_gen_acc+=tr_gen_acc_part;
train_writer.add_summary(summary,i*len(total_data)+j*FLAGS.batch_size);
print(str(i+1)+" Epoch Training Cross Entropy: "+str(tr_ce/(j+1))+", Infinity Norm: "+str(tr_infi/(j+1))+",Gen Acc: "+str(tr_gen_acc/(j+1)));
total_idx=range(len(total_data)); np.random.shuffle(total_idx);
total_data=total_data[total_idx];total_label=total_label[total_idx];
saver_gen.save(sess,"mark3_iv3_rv250_rv2152_rv2101_iv4_avg2/my-model_"+str(FLAGS.max_epsilon)+".ckpt");
def Evaluator(x, y):
num_classes = 1001
# should keep original x here for output
with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
logits_v3, end_points_v3 = inception_v3.inception_v3(
x, num_classes=num_classes, is_training=False)
with slim.arg_scope(inception_v4.inception_v4_arg_scope()):
logits_v4, end_points_v4 = inception_v4.inception_v4(
x, num_classes=num_classes, is_training=False)
with slim.arg_scope(inception_resnet_v2.inception_resnet_v2_arg_scope()):
logits_res_v2, end_points_res_v2 = inception_resnet_v2.inception_resnet_v2(
x, num_classes=num_classes, is_training=False, reuse=True)
with slim.arg_scope(resnet_v2.resnet_arg_scope()):
logits_resnet, end_points_resnet = resnet_v2.resnet_v2_152(
x, num_classes=num_classes, is_training=False)
with slim.arg_scope(resnet_v2.resnet_arg_scope()):
logits_resnet_101, end_points_resnet_101 = resnet_v2.resnet_v2_101(
x, num_classes=num_classes, is_training=False, reuse=tf.AUTO_REUSE)
with slim.arg_scope(resnet_v2.resnet_arg_scope()):
logits_resnet_50, end_points_resnet_50 = resnet_v2.resnet_v2_50(
x, num_classes=num_classes, is_training=False, reuse=tf.AUTO_REUSE)
with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
logits_adv_v3, end_points_adv_v3 = inception_v3.inception_v3(
x,
num_classes=num_classes,
is_training=False,
scope='AdvInceptionV3')
with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
logits_ens3_adv_v3, end_points_ens3_adv_v3 = inception_v3.inception_v3(
x,
num_classes=num_classes,
is_training=False,
scope='Ens3AdvInceptionV3')
with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
logits_ens4_adv_v3, end_points_ens4_adv_v3 = inception_v3.inception_v3(
x,
num_classes=num_classes,
is_training=False,
scope='Ens4AdvInceptionV3')
with slim.arg_scope(inception_resnet_v2.inception_resnet_v2_arg_scope()):
logits_ensadv_res_v2, end_points_ensadv_res_v2 = inception_resnet_v2.inception_resnet_v2(
x,
num_classes=num_classes,
is_training=False,
scope='EnsAdvInceptionResnetV2')
# with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
# logits_ens_v3, end_points_ens_v3 = inception_v3.inception_v3(
# x, num_classes=num_classes, is_training=False)
# acc_v3 = tf.reduce_sum(tf.cast(tf.equal(tf.argmax(end_points_v3['Predictions'], 1), y), tf.float32))
# acc_v4 = tf.reduce_sum(tf.cast(tf.equal(tf.argmax(end_points_v4['Predictions'], 1), y), tf.float32))
# acc_res_v2 = tf.reduce_sum(tf.cast(tf.equal(tf.argmax(end_points_res_v2['Predictions'], 1), y), tf.float32))
# acc_resnet = tf.reduce_sum(tf.cast(tf.equal(tf.argmax(end_points_resnet['predictions'], 1), y), tf.float32))
# acc_resnet_50 = tf.reduce_sum(tf.cast(tf.equal(tf.argmax(end_points_resnet_50['predictions'], 1), y), tf.float32))
# acc_resnet_101 = tf.reduce_sum(tf.cast(tf.equal(tf.argmax(end_points_resnet_101['predictions'], 1), y), tf.float32))
# acc_ens_v3 = tf.reduce_sum(tf.cast(tf.equal(tf.argmax(end_points_ens_v3['predictions'], 1), y), tf.float32))
logits_esm = (logits_v3 + logits_v4 + logits_res_v2 + logits_resnet +
logits_resnet_50 + logits_resnet_101)
if FLAGS.att_model == "incep_v3":
logits_esm = (logits_esm - logits_v3) / 5
elif FLAGS.att_model == "incep_v4":
logits_esm = (logits_esm - logits_v4) / 5
elif FLAGS.att_model == "incep_res_v2":
logits_esm = (logits_esm - logits_res_v2) / 5
elif FLAGS.att_model == "resnet_50":
logits_esm = (logits_esm - logits_resnet_50) / 5
elif FLAGS.att_model == "resnet_101":
logits_esm = (logits_esm - logits_resnet_101) / 5
elif FLAGS.att_model == "resnet_152":
logits_esm = (logits_esm - logits_resnet) / 5
elif FLAGS.att_model == "ens3_adv_3":
logits_esm = (logits_esm + logits_ens4_adv_v3 +
logits_ensadv_res_v2) / 8
elif FLAGS.att_model == "ens4_adv_3":
logits_esm = (logits_esm + logits_ens3_adv_v3 +
logits_ensadv_res_v2) / 8
elif FLAGS.att_model == "ensadv_res_2":
logits_esm = (logits_esm + logits_ens4_adv_v3 + logits_ens3_adv_v3) / 8
# top_k
top_k = FLAGS.top_k
acc_v3 = tf.reduce_sum(
tf.cast(tf.nn.in_top_k(end_points_v3['Predictions'], y, k=top_k),
tf.float32))
acc_v4 = tf.reduce_sum(
tf.cast(tf.nn.in_top_k(end_points_v4['Predictions'], y, k=top_k),
tf.float32))
acc_res_v2 = tf.reduce_sum(
tf.cast(tf.nn.in_top_k(end_points_res_v2['Predictions'], y, k=top_k),
tf.float32))
acc_resnet = tf.reduce_sum(
tf.cast(tf.nn.in_top_k(end_points_resnet['predictions'], y, k=top_k),
tf.float32))
acc_resnet_50 = tf.reduce_sum(
tf.cast(
tf.nn.in_top_k(end_points_resnet_50['predictions'], y, k=top_k),
tf.float32))
acc_resnet_101 = tf.reduce_sum(
tf.cast(
tf.nn.in_top_k(end_points_resnet_101['predictions'], y, k=top_k),
tf.float32))
acc_adv_v3 = tf.reduce_sum(
tf.cast(tf.nn.in_top_k(end_points_adv_v3['Predictions'], y, k=top_k),
tf.float32))
acc_ens3_adv_v3 = tf.reduce_sum(
tf.cast(
tf.nn.in_top_k(end_points_ens3_adv_v3['Predictions'], y, k=top_k),
tf.float32))
acc_ens4_adv_v3 = tf.reduce_sum(
tf.cast(
tf.nn.in_top_k(end_points_ens4_adv_v3['Predictions'], y, k=top_k),
tf.float32))
acc_ensadv_res_v2 = tf.reduce_sum(
tf.cast(
tf.nn.in_top_k(end_points_ensadv_res_v2['Predictions'], y,
k=top_k), tf.float32))
acc_esm = tf.reduce_sum(
tf.cast(
tf.nn.in_top_k(slim.softmax(logits_esm, scope='predictions'),
y,
k=top_k), tf.float32))
# acc_ens_v3 = tf.reduce_sum(tf.cast(tf.equal(tf.argmax(end_points_ens_v3['predictions'], 1), y), tf.float32))
# pred1 = tf.argmax(end_points_v3['Predictions'], 1)
# pred2 = tf.argmax(end_points_v4['Predictions'], 1)
# pred3 = tf.argmax(end_points_res_v2['Predictions'], 1)
# pred4 = tf.argmax(end_points_resnet['predictions'], 1)
return acc_v3, acc_v4, acc_res_v2, acc_resnet, acc_resnet_50, acc_resnet_101, acc_adv_v3, acc_ens3_adv_v3, \
acc_ens4_adv_v3, acc_ensadv_res_v2, acc_esm, end_points_v3, end_points_v4, end_points_res_v2, end_points_resnet
def run_training(path_db,
pid,
category,
task_id,
path_unknown,
pretrained_dir,
tensorflow_dir,
path_save,
num_epochs=1000,
batch_size=32,
finetune_last_layer=False,
data_augmentation=True,
mix_up=False,
network_model='inception-v3',
restore_all_parameters=False,
initial_learning_rate=0.0002,
learning_rate_decay_factor=0.7,
num_epochs_before_decay=2):
##### start parameters for creating TFRecord files #####
#validation_size = 0.1
validation_size = 0.0
num_shards = 2
random_seed = 0
##### end parameters for creating TFRecord files #####
dataset_dir = os.path.join(path_db, pid, category)
log_dir = path_save
tfrecord_filename = pid + '_' + category
if _dataset_exists(dataset_dir=dataset_dir,
_NUM_SHARDS=num_shards,
output_filename=tfrecord_filename):
print('Dataset files already exist. Overwrite them.')
photo_filenames, class_names = _get_filenames_and_classes(
dataset_dir, path_unknown)
# dictionary for class name and class ID
class_names_to_ids = dict(zip(class_names, range(len(class_names))))
# number of validation examples
num_validation = int(validation_size * len(photo_filenames))
# divide to training and validation data
random.seed(random_seed)
random.shuffle(photo_filenames)
training_filenames = photo_filenames[num_validation:]
validation_filenames = photo_filenames[:num_validation]
# find available GPU ID
gpu_id = gpu_utils.pick_gpu_lowest_memory()
# if log directory does not exist, create log directory and dataset
if not os.path.exists(log_dir):
os.makedirs(log_dir)
print('found lowest memory gpu id : ' + str(gpu_id))
_convert_dataset(gpu_id,
'train',
training_filenames,
class_names_to_ids,
dataset_dir=dataset_dir,
tfrecord_filename=tfrecord_filename,
_NUM_SHARDS=num_shards)
_convert_dataset(gpu_id,
'validation',
validation_filenames,
class_names_to_ids,
dataset_dir=dataset_dir,
tfrecord_filename=tfrecord_filename,
_NUM_SHARDS=num_shards)
labels_to_class_names = dict(zip(range(len(class_names)), class_names))
write_label_file(labels_to_class_names, dataset_dir)
print('finished creating dataset ' + tfrecord_filename)
# start training
output_label_filepath = os.path.join(dataset_dir, 'labels.txt')
if network_model!='inception-v4' and network_model!='inception-v3' and network_model!='resnet-v2-50' and network_model!='resnet-v2-152' and \
network_model!='vgg-16' and network_model!='mobilenet-v1' and network_model!='nasnet-large' and network_model!='nasnet-mobile':
print("invalid network model : " + network_model)
sys.exit()
# find pretrained model
if os.path.exists(os.path.join(log_dir, 'model.ckpt')):
checkpoint_file = os.path.join(log_dir, 'model.ckpt')
else:
if network_model == 'inception-v4':
checkpoint_file = os.path.join(
pretrained_dir, 'inception_resnet_v2_2016_08_30.ckpt')
elif network_model == 'inception-v3':
checkpoint_file = os.path.join(pretrained_dir, 'inception_v3.ckpt')
elif network_model == 'resnet-v2-50':
checkpoint_file = os.path.join(pretrained_dir, 'resnet_v2_50.ckpt')
elif network_model == 'resnet-v2-152':
checkpoint_file = os.path.join(pretrained_dir,
'resnet_v2_152.ckpt')
elif network_model == 'vgg-16':
checkpoint_file = os.path.join(pretrained_dir, 'vgg_16.ckpt')
elif network_model == 'mobilenet-v1':
checkpoint_file = os.path.join(pretrained_dir,
'mobilenet_v1_1.0_224.ckpt')
elif network_model == 'nasnet-large':
checkpoint_file = os.path.join(pretrained_dir,
'nasnet-a_large_04_10_2017',
'model.ckpt')
elif network_model == 'nasnet-mobile':
checkpoint_file = os.path.join(pretrained_dir,
'nasnet-a_mobile_04_10_2017',
'model.ckpt')
else:
print("invalid network model : " + network_model)
sys.exit()
# set image size
if network_model == 'inception-v4' or network_model == 'inception-v3' or network_model == 'resnet-v2-50' or network_model == 'resnet-v2-152':
image_size = 299
elif network_model == 'vgg-16' or network_model == 'mobilenet-v1' or network_model == 'nasnet-mobile':
image_size = 224
elif network_model == 'nasnet-large':
image_size = 331
else:
print("invalid network model : " + network_model)
sys.exit()
# create the file pattern of TFRecord files
file_pattern = tfrecord_filename + '_%s_*.tfrecord'
file_pattern_for_counting = tfrecord_filename
labels_to_name, label_list = load_labels(output_label_filepath)
num_classes = len(label_list)
# create a dataset discription
items_to_descriptions = {
'image':
'A 3-channel RGB coloured image that is either ' +
','.join(label_list),
'label':
'A label that is as such -- ' + ','.join([
str(key) + ':' + labels_to_name[key]
for key in labels_to_name.keys()
])
}
# start training
with tf.Graph().as_default() as graph:
tf.logging.set_verbosity(tf.logging.INFO)
# create dataset and load one batch
dataset = get_split('train', dataset_dir, file_pattern,
file_pattern_for_counting, labels_to_name,
num_classes, items_to_descriptions)
images, _, labels = load_batch(dataset,
batch_size=batch_size,
data_augmentation=data_augmentation,
mix_up=mix_up,
height=image_size,
width=image_size)
# number of steps to take before decaying the learning rate and batches per epoch
num_batches_per_epoch = int(dataset.num_samples / batch_size)
num_steps_per_epoch = num_batches_per_epoch # because one step is one batch processed
decay_steps = int(num_epochs_before_decay * num_steps_per_epoch)
# create model for inference
finetune_vars = []
if network_model == 'inception-v4':
with slim.arg_scope(inception_resnet_v2_arg_scope()):
logits, end_points = inception_resnet_v2(
images, num_classes=dataset.num_classes, is_training=True)
finetune_vars = [
'InceptionResnetV2/Logits', 'InceptionResnetV2/AuxLogits'
]
elif network_model == 'inception-v3':
with slim.arg_scope(inception_v3_arg_scope()):
logits, end_points = inception_v3(
images, num_classes=dataset.num_classes, is_training=True)
finetune_vars = ['InceptionV3/Logits', 'InceptionV3/AuxLogits']
elif network_model == 'resnet-v2-50':
with slim.arg_scope(resnet_arg_scope()):
logits, end_points = resnet_v2_50(
images, num_classes=dataset.num_classes, is_training=True)
finetune_vars = ['resnet_v2_50/logits']
elif network_model == 'resnet-v2-152':
with slim.arg_scope(resnet_arg_scope()):
logits, end_points = resnet_v2_152(
images, num_classes=dataset.num_classes, is_training=True)
finetune_vars = ['resnet_v2_152/logits']
elif network_model == 'vgg-16':
with slim.arg_scope(vgg_arg_scope()):
logits, _ = vgg_16(images,
num_classes=dataset.num_classes,
is_training=True)
finetune_vars = ['vgg_16/fc8']
elif network_model == 'mobilenet-v1':
with slim.arg_scope(mobilenet_v1_arg_scope()):
logits, end_points = mobilenet_v1(
images, num_classes=dataset.num_classes, is_training=True)
finetune_vars = ['MobilenetV1/Logits']
elif network_model == 'nasnet-large':
with slim.arg_scope(nasnet.nasnet_large_arg_scope()):
logits, end_points = nasnet.build_nasnet_large(
images, dataset.num_classes)
finetune_vars = [
'final_layer', 'aux_11',
'cell_stem_0/comb_iter_0/left/global_step'
]
elif network_model == 'nasnet-mobile':
with slim.arg_scope(nasnet.nasnet_mobile_arg_scope()):
logits, end_points = nasnet.build_nasnet_mobile(
images, dataset.num_classes)
finetune_vars = ['final_layer', 'aux_7']
else:
print("Invalid network model : " + network_model)
sys.exit()
# define the scopes that you want to exclude for restoration
exclude = []
if not restore_all_parameters:
exclude = finetune_vars
variables_to_restore = slim.get_variables_to_restore(exclude=exclude)
if mix_up:
labels.set_shape([batch_size, dataset.num_classes])
logits.set_shape([batch_size, dataset.num_classes])
loss = tf.losses.sigmoid_cross_entropy(labels, logits)
else:
# perform one-hot-encoding of the labels (Try one-hot-encoding within the load_batch function!)
one_hot_labels = slim.one_hot_encoding(labels, dataset.num_classes)
# performs the equivalent to tf.nn.sparse_softmax_cross_entropy_with_logits but enhanced with checks
loss = tf.losses.softmax_cross_entropy(
onehot_labels=one_hot_labels, logits=logits)
total_loss = tf.losses.get_total_loss(
) #obtain the regularization losses as well
# create the global step for monitoring the learning_rate and training.
global_step = tf.train.get_or_create_global_step()
# define your exponentially decaying learning rate
lr = tf.train.exponential_decay(learning_rate=initial_learning_rate,
global_step=global_step,
decay_steps=decay_steps,
decay_rate=learning_rate_decay_factor,
staircase=True)
# define optimizer
optimizer = tf.train.AdamOptimizer(learning_rate=lr)
# create train_op
if finetune_last_layer:
variables_to_train = get_variables_to_train_by_scopes(
finetune_vars)
print("finetune variables : " + str(variables_to_train))
train_op = slim.learning.create_train_op(
total_loss, optimizer, variables_to_train=variables_to_train)
else:
train_op = slim.learning.create_train_op(total_loss, optimizer)
# define prediction matrix
if network_model=='inception-v4' or network_model=='inception-v3' or network_model=='mobilenet-v1' or \
network_model=='nasnet-large' or network_model=='nasnet-mobile':
predictions = tf.argmax(end_points['Predictions'], 1)
probabilities = end_points['Predictions']
elif network_model == 'resnet-v2-50' or network_model == 'resnet-v2-152':
predictions = tf.argmax(end_points['predictions'], 1)
probabilities = end_points['predictions']
elif network_model == 'vgg-16':
predictions = tf.argmax(logits, 1)
probabilities = tf.nn.softmax(logits)
else:
print("Invalid network model : " + network_model)
sys.exit()
if mix_up:
argmax_labels = tf.argmax(labels, 1)
accuracy, accuracy_update = tf.contrib.metrics.streaming_accuracy(
predictions, argmax_labels)
else:
accuracy, accuracy_update = tf.contrib.metrics.streaming_accuracy(
predictions, labels)
metrics_op = tf.group(accuracy_update, probabilities)
# create summaries
tf.summary.scalar('losses/Total_Loss', total_loss)
tf.summary.scalar('accuracy', accuracy)
tf.summary.scalar('learning_rate', lr)
my_summary_op = tf.summary.merge_all()
# defube training step function that runs both the train_op, metrics_op and updates the global_step concurrently
def train_step(sess, train_op, global_step):
# check the time for each sess run
start_time = time.time()
total_loss, global_step_count, _ = sess.run(
[train_op, global_step, metrics_op])
time_elapsed = time.time() - start_time
# run the logging to print some results
logging.info('global step %s: loss: %.4f (%.2f sec/step)',
global_step_count, total_loss, time_elapsed)
return total_loss, int(global_step_count)
# create a saver function that actually restores the variables from a checkpoint file
saver = tf.train.Saver(variables_to_restore)
def restore_fn(sess):
return saver.restore(sess, checkpoint_file)
# define your supervisor for running a managed session
sv = tf.train.Supervisor(logdir=log_dir,
summary_op=None,
init_fn=restore_fn)
# run the managed session
start_train_time = time.time()
gpu_options = tf.ConfigProto(
gpu_options=tf.GPUOptions(visible_device_list=str(gpu_id),
per_process_gpu_memory_fraction=0.4))
with sv.prepare_or_wait_for_session(config=gpu_options) as sess:
for step in range(num_steps_per_epoch * num_epochs):
# check if training task is not canceled
if not controller.check_train_task_alive(
pid, category, task_id):
print('Training task is canceled.')
sv.stop()
return False, "", "", output_label_filepath, global_step_count
# at the start of every epoch, show the vital information:
if step % num_batches_per_epoch == 0:
logging.info('Epoch %s/%s',
step / num_batches_per_epoch + 1, num_epochs)
learning_rate_value, accuracy_value = sess.run(
[lr, accuracy])
logging.info('Current Learning Rate: %s',
learning_rate_value)
logging.info('Current Streaming Accuracy: %s',
accuracy_value)
# optionally, print your logits and predictions for a sanity check that things are going fine.
logits_value, probabilities_value, predictions_value, labels_value = sess.run(
[logits, probabilities, predictions, labels])
print('logits: \n', logits_value)
print('Probabilities: \n', probabilities_value)
print('predictions: \n', predictions_value)
print('Labels:\n:', labels_value)
# log the summaries every 10 step.
if step % 10 == 0:
loss, global_step_count = train_step(
sess, train_op, sv.global_step)
summaries = sess.run(my_summary_op)
sv.summary_computed(sess, summaries)
# if not, simply run the training step
else:
loss, global_step_count = train_step(
sess, train_op, sv.global_step)
# if specific time passes, save model for evaluation
time_elapsed_train = time.time() - start_train_time
print('training time : ' + str(time_elapsed_train))
# log the final training loss and accuracy
logging.info(
'Training Progress : %.2f %% ',
100.0 * step / float(num_steps_per_epoch * num_epochs))
logging.info('Final Loss: %s', loss)
logging.info('Global Step: %s', global_step_count)
logging.info('Final Accuracy: %s', sess.run(accuracy))
# after all the training has been done, save the log files and checkpoint model
logging.info('Finished training! Saving model to disk now.')
sv.saver.save(sess, sv.save_path, global_step=sv.global_step)
# save graph definition file
output_graph_filepath = os.path.join(log_dir, 'graph.pb')
export_graph_command_exec = "./network/export_slim_graph.py"
if not os.path.exists(export_graph_command_exec):
print("fatal error, cannot find command : " +
export_graph_command_exec)
sys.exit()
export_graph_command_env = os.environ.copy()
export_graph_command_env["CUDA_VISIBLE_DEVICES"] = ''
export_graph_command = []
export_graph_command.append(sys.executable)
export_graph_command.append(export_graph_command_exec)
export_graph_command.append(network_model)
export_graph_command.append(str(dataset.num_classes))
export_graph_command.append(output_graph_filepath)
print("start exec:" + " ".join(export_graph_command))
proc = subprocess.Popen(export_graph_command,
env=export_graph_command_env)
print("export graph process ID=" + str(proc.pid))
controller.upsert_train_child_process(task_id, proc.pid)
proc.communicate()
controller.delete_train_child_process(task_id, proc.pid)
print("finish exec:" + " ".join(export_graph_command))
if not controller.check_train_task_alive(pid, category, task_id):
print('Training task is canceled.')
sv.stop()
return False, "", "", output_label_filepath, global_step_count
# save frozon graph, optimized graph, and quantized graph from graph definition and checkpoint
latest_checkpoint_filepath = tf.train.latest_checkpoint(log_dir)
# you can check output node name by tensorflow/tools/graph_transforms::summarize_graph
# https://github.com/tensorflow/models/tree/master/research/slim#Export
output_node_names = ""
if network_model == 'inception-v4':
output_node_names = "InceptionResnetV2/Logits/Predictions"
elif network_model == 'inception-v3':
output_node_names = "InceptionV3/AuxLogits/SpatialSqueeze,InceptionV3/Predictions/Reshape_1"
elif network_model == 'resnet-v2-50':
output_node_names = "resnet_v2_50/predictions/Reshape_1"
elif network_model == 'resnet-v2-152':
output_node_names = "resnet_v2_152/predictions/Reshape_1"
elif network_model == 'vgg-16':
output_node_names = "vgg_16/fc8/squeezed"
elif network_model == 'mobilenet-v1':
output_node_names = "MobilenetV1/Predictions/Reshape_1"
elif network_model == 'nasnet-large' or network_model == 'nasnet-mobile':
output_node_names = "final_layer/predictions"
else:
print("Invalid network model : " + network_model)
sys.exit()
output_frozen_graph_filepath = os.path.join(
log_dir, 'frozen_graph.pb')
freeze_graph_command_exec = os.path.join(
tensorflow_dir,
"bazel-bin/tensorflow/python/tools/freeze_graph")
if not os.path.exists(freeze_graph_command_exec):
print("fatal error, cannot find command : " +
freeze_graph_command_exec)
sys.exit()
freeze_graph_command_env = os.environ.copy()
freeze_graph_command_env["CUDA_VISIBLE_DEVICES"] = ''
freeze_graph_command = []
freeze_graph_command.append(freeze_graph_command_exec)
freeze_graph_command.append("--input_graph=" +
output_graph_filepath)
freeze_graph_command.append("--input_checkpoint=" +
latest_checkpoint_filepath)
freeze_graph_command.append("--input_binary=true")
freeze_graph_command.append("--output_graph=" +
output_frozen_graph_filepath)
freeze_graph_command.append("--output_node_names=" +
output_node_names)
print("start exec:" + " ".join(freeze_graph_command))
proc = subprocess.Popen(freeze_graph_command,
env=freeze_graph_command_env)
print("freeze graph process ID=" + str(proc.pid))
controller.upsert_train_child_process(task_id, proc.pid)
proc.communicate()
controller.delete_train_child_process(task_id, proc.pid)
print("finish exec:" + " ".join(freeze_graph_command))
if not controller.check_train_task_alive(pid, category, task_id):
print('Training task is canceled.')
sv.stop()
return False, "", "", output_label_filepath, global_step_count
output_optimized_graph_filepath = os.path.join(
log_dir, 'optimized_graph.pb')
optimize_graph_command_exec = os.path.join(
tensorflow_dir,
"bazel-bin/tensorflow/python/tools/optimize_for_inference")
if not os.path.exists(optimize_graph_command_exec):
print("fatal error, cannot find command : " +
optimize_graph_command_exec)
sys.exit()
optimize_graph_command_env = os.environ.copy()
optimize_graph_command_env["CUDA_VISIBLE_DEVICES"] = ''
optimize_graph_command = []
optimize_graph_command.append(optimize_graph_command_exec)
optimize_graph_command.append("--input=" +
output_frozen_graph_filepath)
optimize_graph_command.append("--output=" +
output_optimized_graph_filepath)
optimize_graph_command.append("--input_names=input")
optimize_graph_command.append("--output_names=" +
output_node_names)
optimize_graph_command.append("--frozen_graph=true")
print("start exec:" + " ".join(optimize_graph_command))
proc = subprocess.Popen(optimize_graph_command,
env=optimize_graph_command_env)
print("optimize graph process ID=" + str(proc.pid))
controller.upsert_train_child_process(task_id, proc.pid)
proc.communicate()
controller.delete_train_child_process(task_id, proc.pid)
print("finish exec:" + " ".join(optimize_graph_command))
if not controller.check_train_task_alive(pid, category, task_id):
print('Training task is canceled.')
sv.stop()
return False, "", "", output_label_filepath, global_step_count
output_quantized_graph_filepath = os.path.join(
log_dir, 'quantized_graph.pb')
quantize_graph_command_exec = os.path.join(
tensorflow_dir,
"bazel-bin/tensorflow/tools/quantization/quantize_graph")
if not os.path.exists(quantize_graph_command_exec):
print("fatal error, cannot find command : " +
quantize_graph_command_exec)
sys.exit()
quantize_graph_command_env = os.environ.copy()
quantize_graph_command_env["CUDA_VISIBLE_DEVICES"] = ''
quantize_graph_command = []
quantize_graph_command.append(quantize_graph_command_exec)
quantize_graph_command.append("--input=" +
output_optimized_graph_filepath)
quantize_graph_command.append("--output=" +
output_quantized_graph_filepath)
quantize_graph_command.append("--input_node_names=input")
quantize_graph_command.append("--output_node_names=" +
output_node_names)
quantize_graph_command.append("--mode=eightbit")
print("start exec:" + " ".join(quantize_graph_command))
proc = subprocess.Popen(quantize_graph_command,
env=quantize_graph_command_env)
print("quantize graph process ID=" + str(proc.pid))
controller.upsert_train_child_process(task_id, proc.pid)
proc.communicate()
controller.delete_train_child_process(task_id, proc.pid)
print("finish exec:" + " ".join(quantize_graph_command))
if not controller.check_train_task_alive(pid, category, task_id):
print('Training task is canceled.')
sv.stop()
return False, "", "", output_label_filepath, global_step_count
return True, output_optimized_graph_filepath, output_quantized_graph_filepath, output_label_filepath, global_step_count
def graph(x, adv, y, t_y, i, x_max, x_min, grad, amplification):
target_one_hot = tf.one_hot(t_y, 1001)
true_one_hot = tf.one_hot(y, 1001)
eps = 2.0 * FLAGS.max_epsilon / 255.0
num_iter = FLAGS.num_iter
alpha = eps / num_iter
alpha_beta = alpha * 10.0
gamma = alpha_beta * 0.8
momentum = FLAGS.momentum
num_classes = 1001
# input_diversity(FLAGS, adv): for DI-FGSM
with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
logits_v3, end_points_v3 = inception_v3.inception_v3(
input_diversity(FLAGS, adv), num_classes=num_classes, is_training=False, reuse=True)
auxlogit_v3 = end_points_v3['AuxLogits']
with slim.arg_scope(inception_v4.inception_v4_arg_scope()):
logits_v4, end_points_v4 = inception_v4.inception_v4(
input_diversity(FLAGS, adv), num_classes=num_classes, is_training=False, reuse=True)
auxlogit_v4 = end_points_v4['AuxLogits']
with slim.arg_scope(resnet_v2.resnet_arg_scope()):
logits_resnet_152, end_points_resnet = resnet_v2.resnet_v2_152(
input_diversity(FLAGS, adv), num_classes=num_classes, is_training=False, reuse=True)
with slim.arg_scope(resnet_v2.resnet_arg_scope()):
logits_resnet_101, end_points_resnet_101 = resnet_v2.resnet_v2_101(
input_diversity(FLAGS, adv), num_classes=num_classes, is_training=False, reuse=True)
with slim.arg_scope(resnet_v2.resnet_arg_scope()):
logits_resnet_50, end_points_resnet_50 = resnet_v2.resnet_v2_50(
input_diversity(FLAGS, adv), num_classes=num_classes, is_training=False, reuse=True)
with slim.arg_scope(inception_resnet_v2.inception_resnet_v2_arg_scope()):
logits_Incres, end_points_IR = inception_resnet_v2.inception_resnet_v2(
input_diversity(FLAGS, adv), num_classes=num_classes, is_training=False, reuse=True)
auxlogit_IR = end_points_IR['AuxLogits']
with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
logits_ens3_adv_v3, end_points_ens3_adv_v3 = inception_v3.inception_v3(
input_diversity(FLAGS, adv), num_classes=num_classes, is_training=False, scope='Ens3AdvInceptionV3',reuse=True)
auxlogit_ens3 = end_points_ens3_adv_v3['AuxLogits']
with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
logits_ens4_adv_v3, end_points_ens4_adv_v3 = inception_v3.inception_v3(
input_diversity(FLAGS, adv), num_classes=num_classes, is_training=False, scope='Ens4AdvInceptionV3', reuse=True)
auxlogit_ens4 = end_points_ens4_adv_v3['AuxLogits']
# with slim.arg_scope(inception_resnet_v2.inception_resnet_v2_arg_scope()):
# logits_ensadv_res_v2, end_points_ensadv_res_v2 = inception_resnet_v2.inception_resnet_v2(
# input_diversity(FLAGS, adv), num_classes=num_classes, is_training=False, scope='EnsAdvInceptionResnetV2', reuse=True)
# auxlogit_resv2 = end_points_ensadv_res_v2['AuxLogits']
# FLAGS.temperature: for PI-FGSM++
logits = (logits_v4 + logits_resnet_152 + logits_v3 + logits_resnet_101 + logits_resnet_50 + \
logits_Incres + logits_ens3_adv_v3 + logits_ens4_adv_v3) / 8.0 / FLAGS.temperature
auxlogits = (auxlogit_v4 + auxlogit_v3 + auxlogit_IR + auxlogit_ens3 + auxlogit_ens4) / 5.0 / FLAGS.temperature
target_cross_entropy = tf.losses.softmax_cross_entropy(target_one_hot,
logits,
label_smoothing=0.0,
weights=1.0)
target_cross_entropy += tf.losses.softmax_cross_entropy(target_one_hot,
auxlogits,
label_smoothing=0.0,
weights=1.0)
noise = tf.gradients(target_cross_entropy, adv)[0]
noise = tf.nn.depthwise_conv2d(noise, T_kern, strides=[1, 1, 1, 1], padding='SAME')
# for MI-FGSM
# noise = noise / tf.reduce_mean(tf.abs(noise), [1, 2, 3], keep_dims=True)
# noise = momentum * grad + noise
amplification += alpha_beta * n_staircase_sign(noise, num_of_K)
cut_noise = tf.clip_by_value(abs(amplification) - eps, 0.0, 10000.0) * tf.sign(amplification)
projection = gamma * n_staircase_sign(project_noise(cut_noise, P_kern, kern_size), num_of_K)
adv = adv - alpha_beta * n_staircase_sign(noise, num_of_K) - projection
# adv = adv - alpha * n_staircase_sign(noise, num_of_K)
adv = tf.clip_by_value(adv, x_min, x_max)
i = tf.add(i, 1)
return x, adv, y, t_y, i, x_max, x_min, noise, amplification,
def main(_):
# Images for inception classifier are normalized to be in [-1, 1] interval,
# eps is a difference between pixels so it should be in [0, 2] interval.
# Renormalizing epsilon from [0, 255] to [0, 2].
eps = 2.0 * FLAGS.max_epsilon / 255.0
num_classes = 1001
batch_shape = [FLAGS.batch_size, FLAGS.image_height, FLAGS.image_width, 3]
tf.logging.set_verbosity(tf.logging.INFO)
with tf.Graph().as_default():
tf.set_random_seed(305)
# Prepare graph
x_input = tf.placeholder(tf.float32, shape = batch_shape)
adv_img = tf.placeholder(tf.float32, shape = batch_shape)
y = tf.placeholder(tf.int32, shape = batch_shape[0])
t_y = tf.placeholder(tf.int32, shape = batch_shape[0])
x_max = tf.clip_by_value(x_input + eps, -1.0, 1.0)
x_min = tf.clip_by_value(x_input - eps, -1.0, 1.0)
with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
logits_v3, end_points_v3 = inception_v3.inception_v3(
adv_img, num_classes=num_classes, is_training=False)
pre_v3 = tf.argmax(logits_v3, 1)
with slim.arg_scope(inception_v4.inception_v4_arg_scope()):
logits_v4, end_points_v4 = inception_v4.inception_v4(
adv_img, num_classes=num_classes, is_training=False)
pre_v4 = tf.argmax(logits_v4, 1)
with slim.arg_scope(resnet_v2.resnet_arg_scope()):
logits_resnet_152, end_points_resnet = resnet_v2.resnet_v2_152(
adv_img, num_classes=num_classes, is_training=False)
pre_resnet_152 = tf.argmax(logits_resnet_152, 1)
with slim.arg_scope(resnet_v2.resnet_arg_scope()):
logits_resnet_101, end_points_resnet_101 = resnet_v2.resnet_v2_101(
adv_img, num_classes=num_classes, is_training=False)
pre_resnet_101 = tf.argmax(logits_resnet_101, 1)
with slim.arg_scope(resnet_v2.resnet_arg_scope()):
logits_resnet_50, end_points_resnet_50 = resnet_v2.resnet_v2_50(
adv_img, num_classes=num_classes, is_training=False)
pre_resnet_50 = tf.argmax(logits_resnet_50, 1)
with slim.arg_scope(inception_resnet_v2.inception_resnet_v2_arg_scope()):
logits_Incres, end_points_IR = inception_resnet_v2.inception_resnet_v2(
adv_img, num_classes=num_classes, is_training=False)
pre_Inc_res = tf.argmax(logits_Incres, 1)
with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
logits_ens3_adv_v3, end_points_ens3_adv_v3 = inception_v3.inception_v3(
adv_img, num_classes=num_classes, is_training=False, scope='Ens3AdvInceptionV3')
pre_ens3_adv_v3 = tf.argmax(logits_ens3_adv_v3, 1)
with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
logits_ens4_adv_v3, end_points_ens4_adv_v3 = inception_v3.inception_v3(
adv_img, num_classes=num_classes, is_training=False, scope='Ens4AdvInceptionV3')
pre_ens4_adv_v3 = tf.argmax(logits_ens4_adv_v3, 1)
with slim.arg_scope(inception_resnet_v2.inception_resnet_v2_arg_scope()):
logits_ensadv_res_v2, end_points_ensadv_res_v2 = inception_resnet_v2.inception_resnet_v2(
adv_img, num_classes=num_classes, is_training=False, scope='EnsAdvInceptionResnetV2')
pre_ensadv_res_v2 = tf.argmax(logits_ensadv_res_v2, 1)
pre_ensemble_logit = tf.argmax((logits_resnet_152 + logits_v4 + logits_v3 + + logits_resnet_101 + logits_resnet_50 +\
logits_Incres + logits_ens3_adv_v3 + logits_ens4_adv_v3), 1)
mean_pert = 0.0
sum_v3, sum_v4, sum_res152, sum_res101, sum_res50, sum_Incres, sum_ensemble = 0, 0, 0, 0, 0, 0, 0
sum_ens3_adv_v3, sum_ens4_adv_v3, sum_ensadv_res_v2 = 0, 0, 0
i = tf.constant(0)
grad = tf.zeros(shape=batch_shape)
amplification = tf.zeros(shape=batch_shape)
_, x_adv, _, _, _, _, _, _, _ = tf.while_loop(stop, graph, [x_input, adv_img, y, t_y, i, x_max, x_min, grad, amplification])
# Run computation
s1 = tf.train.Saver(slim.get_model_variables(scope='InceptionV3'))
s3 = tf.train.Saver(slim.get_model_variables(scope='Ens3AdvInceptionV3'))
s4 = tf.train.Saver(slim.get_model_variables(scope='Ens4AdvInceptionV3'))
s5 = tf.train.Saver(slim.get_model_variables(scope='InceptionV4'))
s6 = tf.train.Saver(slim.get_model_variables(scope='InceptionResnetV2'))
s7 = tf.train.Saver(slim.get_model_variables(scope='EnsAdvInceptionResnetV2'))
s8 = tf.train.Saver(slim.get_model_variables(scope='resnet_v2_152'))
s9 = tf.train.Saver(slim.get_model_variables(scope='resnet_v2_101'))
s10 = tf.train.Saver(slim.get_model_variables(scope='resnet_v2_50'))
with tf.Session() as sess:
s1.restore(sess, model_checkpoint_map['inception_v3'])
s3.restore(sess, model_checkpoint_map['ens3_adv_inception_v3'])
s4.restore(sess, model_checkpoint_map['ens4_adv_inception_v3'])
s5.restore(sess, model_checkpoint_map['inception_v4'])
s6.restore(sess, model_checkpoint_map['inception_resnet_v2'])
s7.restore(sess, model_checkpoint_map['ens_adv_inception_resnet_v2'])
s8.restore(sess, model_checkpoint_map['resnet_v2_152'])
s9.restore(sess, model_checkpoint_map['resnet_v2_101'])
s10.restore(sess, model_checkpoint_map['resnet_v2_50'])
import pandas as pd
dev = pd.read_csv(FLAGS.input_csv)
for idx in tqdm(range(0, 1000 // FLAGS.batch_size)):
images, filenames, True_label, Target_label = load_images(FLAGS.input_dir, dev, idx * FLAGS.batch_size, batch_shape)
my_adv_images = sess.run(x_adv, feed_dict={x_input: images, adv_img: images, y: True_label, t_y: Target_label}).astype(np.float32)
mean_pert += abs(my_adv_images - images).mean()
pre_v3_, pre_v4_, pre_resnet152_, pre_resnet101_, pre_resnet50_, pre_Inc_res_,\
pre_ens3_adv_v3_, pre_ens4_adv_v3_, pre_ensadv_res_v2_ ,pre_ensemble_ \
= sess.run([pre_v3, pre_v4, pre_resnet_152, pre_resnet_101, pre_resnet_50, pre_Inc_res,
pre_ens3_adv_v3, pre_ens4_adv_v3, pre_ensadv_res_v2,
pre_ensemble_logit], feed_dict = {adv_img: my_adv_images})
sum_v3 += (pre_v3_ == Target_label).sum()
sum_v4 += (pre_v4_ == Target_label).sum()
sum_res152 += (pre_resnet152_ == Target_label).sum()
sum_res101 += (pre_resnet101_ == Target_label).sum()
sum_res50 += (pre_resnet50_ == Target_label).sum()
sum_Incres += (pre_Inc_res_ == Target_label).sum()
sum_ens3_adv_v3 += (pre_ens3_adv_v3_ == Target_label).sum()
sum_ens4_adv_v3 += (pre_ens4_adv_v3_ == Target_label).sum()
sum_ensadv_res_v2 += (pre_ensadv_res_v2_ == Target_label).sum()
sum_ensemble += (pre_ensemble_ == Target_label).sum()
save_images(my_adv_images, filenames, FLAGS.output_dir)
print('mean noise = ', (mean_pert / (1000.0 / FLAGS.batch_size)) * 255.0)
print('sum_v3 = {}'.format(sum_v3))
print('sum_v4 = {}'.format(sum_v4))
print('sum_res2 = {}'.format(sum_res152))
print('sum_res1 = {}'.format(sum_res101))
print('sum_res1 = {}'.format(sum_res50))
print('sum_Incres_v2 = {}'.format(sum_Incres))
print('sum_ens3_adv_v3 = {}'.format(sum_ens3_adv_v3))
print('sum_ens4_adv_v3 = {}'.format(sum_ens4_adv_v3))
print('sum_ensadv_Incres_v2 = {}'.format(sum_ensadv_res_v2))
print('sum_ensmeble = {}'.format(sum_ensemble))
def graph(x, y, i, x_max, x_min, grad, y_target, y_logits):
eps = 2.0 * FLAGS.max_epsilon / 255.0
num_iter = FLAGS.num_iter
alpha = eps / num_iter
momentum = FLAGS.momentum
num_classes = 1001
# should keep original x here for output
with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
logits_v3, end_points_v3 = inception_v3.inception_v3(
input_diversity(x), num_classes=num_classes, is_training=False)
with slim.arg_scope(inception_v4.inception_v4_arg_scope()):
logits_v4, end_points_v4 = inception_v4.inception_v4(
input_diversity(x), num_classes=num_classes, is_training=False)
with slim.arg_scope(inception_resnet_v2.inception_resnet_v2_arg_scope()):
logits_res_v2, end_points_res_v2 = inception_resnet_v2.inception_resnet_v2(
input_diversity(x), num_classes=num_classes, is_training=False)
with slim.arg_scope(resnet_v2.resnet_arg_scope()):
logits_resnet, end_points_resnet = resnet_v2.resnet_v2_152(
input_diversity(x), num_classes=num_classes, is_training=False, reuse=tf.AUTO_REUSE)
# with slim.arg_scope(resnet_v2.resnet_arg_scope()):
# logits_resnet_50, end_points_resnet_50 = resnet_v2.resnet_v2_50(
# input_diversity(x), num_classes=num_classes, is_training=False, reuse=tf.AUTO_REUSE)
with slim.arg_scope(resnet_v2.resnet_arg_scope()):
logits_resnet_101, end_points_resnet_101 = resnet_v2.resnet_v2_101(
input_diversity(x), num_classes=num_classes, is_training=False, reuse=tf.AUTO_REUSE)
logits = (logits_resnet +
logits_v3 + logits_v4 +
logits_res_v2 + logits_resnet_101 ) / 5
y_oh = tf.one_hot(y, num_classes)
y_target_oh = tf.one_hot(y_target, num_classes)
# loss = -FLAGS.W_crs * tf.losses.softmax_cross_entropy(y_oh,
# logits,
# label_smoothing=0.0,
# weights=1.0)
# loss = - Poincare_dis(tf.clip_by_value((y_oh-0.01), 0.0, 1.0),
# logits / tf.reduce_sum(tf.abs(logits), [1], keep_dims=True) )
loss_ce = tf.losses.softmax_cross_entropy(y_target_oh,
logits,
label_smoothing=0.0,
weights=1.0)
loss_po = Poincare_dis(tf.clip_by_value((y_target_oh-0.00001), 0.0, 1.0),
logits / tf.reduce_sum(tf.abs(logits), [1], keep_dims=True))
loss_cos = tf.clip_by_value((Cos_dis(y_oh, logits) - Cos_dis(y_target_oh, logits) + FLAGS.a), 0.0, 2.1)
# loss_cos = tf.maximum(loss_ce - tf.losses.softmax_cross_entropy(y_oh, logits, label_smoothing=0.0, weights=1.0) + FLAGS.a, 0.0)
if FLAGS.loss == "ce":
loss = loss_ce
elif FLAGS.loss == "po":
loss = loss_po
elif FLAGS.loss == "trip_po":
loss = loss_po + FLAGS.W_cos*loss_cos
# loss += cross_entropy
# loss += FLAGS.W_cos*loss_cos
noise = -tf.gradients(loss, x)[0]
# TI-
noise = tf.nn.depthwise_conv2d(noise, stack_kernel, strides=[1, 1, 1, 1], padding='SAME')
# CE Cross-entry loss must add this term
if FLAGS.loss == 'ce':
noise = noise / tf.reduce_mean(tf.abs(noise), [1, 2, 3], keep_dims=True)
noise = momentum * grad + noise
x = x + alpha * tf.sign(noise)
x = tf.clip_by_value(x, x_min, x_max)
i = tf.add(i, 1)
return x, y, i, x_max, x_min, noise, y_target, logits
def graph(x, mask, y, i, x_max, x_min, grad):
eps = 2.0 * FLAGS.max_epsilon / 255.0
alpha = eps / 10
momentum = FLAGS.momentum
num_classes = 1001
with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
logits_v3, end_points_v3 = inception_v3.inception_v3(
input_diversity(x), num_classes=num_classes, is_training=False)
logits_v3_rotated, _ = inception_v3.inception_v3(
rotate(x), num_classes=num_classes, is_training=False, reuse=True)
with slim.arg_scope(inception_v4.inception_v4_arg_scope()):
logits_v4, end_points_v4 = inception_v4.inception_v4(
input_diversity(x), num_classes=num_classes, is_training=False)
logits_v4_rotated, _ = inception_v4.inception_v4(
rotate(x), num_classes=num_classes, is_training=False, reuse=True)
with slim.arg_scope(inception_resnet_v2.inception_resnet_v2_arg_scope()):
logits_res_v2, end_points_res_v2 = inception_resnet_v2.inception_resnet_v2(
input_diversity(x),
num_classes=num_classes,
is_training=False,
reuse=True)
logits_res_v2_rotated, _ = inception_resnet_v2.inception_resnet_v2(
rotate(x), num_classes=num_classes, is_training=False, reuse=True)
with slim.arg_scope(resnet_v2.resnet_arg_scope()):
logits_resnet, end_points_resnet = resnet_v2.resnet_v2_152(
input_diversity(x), num_classes=num_classes, is_training=False)
logits_resnet_rotated, _ = resnet_v2.resnet_v2_152(
rotate(x), num_classes=num_classes, is_training=False, reuse=True)
with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
logits_ens3_adv_v3, end_points_ens3_adv_v3 = inception_v3.inception_v3(
input_diversity(x),
num_classes=num_classes,
is_training=False,
scope='Ens3AdvInceptionV3')
logits_ens3_adv_v3_rotated, _ = inception_v3.inception_v3(
rotate(x),
num_classes=num_classes,
is_training=False,
scope='Ens3AdvInceptionV3',
reuse=True)
with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
logits_ens4_adv_v3, end_points_ens4_adv_v3 = inception_v3.inception_v3(
input_diversity(x),
num_classes=num_classes,
is_training=False,
scope='Ens4AdvInceptionV3')
logits_ens4_adv_v3_rotated, _ = inception_v3.inception_v3(
rotate(x),
num_classes=num_classes,
is_training=False,
scope='Ens4AdvInceptionV3',
reuse=True)
# with slim.arg_scope(inception_resnet_v2.inception_resnet_v2_arg_scope()):
# logits_ens_advres_v2, end_points_ens_advres_v2 = inception_resnet_v2.inception_resnet_v2(
# input_diversity(x), num_classes=num_classes, is_training=False, scope='EnsAdvInceptionResnetV2')
# logits_ens_advres_v2_rotated, _ = inception_resnet_v2.inception_resnet_v2(
# rotate(x), num_classes=num_classes, is_training=False, scope='EnsAdvInceptionResnetV2', reuse=True)
logits = (logits_v3 + logits_v4 + logits_res_v2 + logits_resnet +
logits_v3_rotated + logits_v4_rotated + logits_res_v2_rotated +
logits_resnet_rotated + logits_ens3_adv_v3 + logits_ens4_adv_v3 +
logits_ens3_adv_v3_rotated + logits_ens4_adv_v3_rotated) / 12
auxlogits = (
end_points_v3['AuxLogits'] + end_points_v4['AuxLogits'] +
end_points_res_v2['AuxLogits'] + end_points_ens3_adv_v3['AuxLogits'] +
end_points_ens4_adv_v3['AuxLogits']
# + end_points_ens_advres_v2['AuxLogits']
) / 5
cross_entropy = tf.losses.softmax_cross_entropy(y,
logits,
label_smoothing=0.0,
weights=1.0)
cross_entropy += tf.losses.softmax_cross_entropy(y,
auxlogits,
label_smoothing=0.0,
weights=0.4)
noise = tf.gradients(cross_entropy, x)[0]
noise = tf.nn.depthwise_conv2d(noise,
stack_kernel,
strides=[1, 1, 1, 1],
padding='SAME')
noise = noise / tf.reduce_mean(tf.abs(noise), [1, 2, 3], keep_dims=True)
noise = momentum * grad + noise
noise = noise * mask[:, :, :, np.newaxis]
x = x + alpha * tf.sign(noise)
x = tf.clip_by_value(x, x_min, x_max)
i = tf.add(i, 1)
return x, mask, y, i, x_max, x_min, noise
def train():
eps = 2.0 * float(FLAGS.max_epsilon) / 256.0
tf.logging.set_verbosity(tf.logging.INFO)
with tf.Graph().as_default():
# Design architecture
# input
x_data = tf.placeholder(tf.float32,
[None, FLAGS.img_height, FLAGS.img_width, 3],
name="x_data")
y_label = tf.placeholder(tf.float32, [None, FLAGS.num_classes],
name="y_label")
# generator
x_generated, g_params = build_generator(x_data, FLAGS)
x_generated = x_generated * eps
x_generated = x_data + x_generated
# discriminator(inception v3)
with slim.arg_scope(inception.inception_v3_arg_scope()):
_, end_points = inception.inception_v3(
x_generated, num_classes=FLAGS.num_classes, is_training=False)
predicted_labels = tf.argmax(end_points['Predictions'], 1)
predicted_logits = end_points['Logits']
disc_var_list = slim.get_model_variables()
# discriminator(resnet v2 50)
x_generated2 = tf.image.resize_bilinear(x_generated, [224, 224],
align_corners=False)
with slim.arg_scope(resnet_utils.resnet_arg_scope()):
_, end_points2 = resnet_v2.resnet_v2_50(
x_generated2, num_classes=FLAGS.num_classes, is_training=False)
predicted_labels2 = tf.argmax(end_points2['predictions'], 1)
predicted_logits2 = end_points2['predictions']
disc_var_list2 = slim.get_model_variables()[len(disc_var_list):]
# discriminator(resnet v2 152)
with slim.arg_scope(resnet_utils.resnet_arg_scope()):
_, end_points3 = resnet_v2.resnet_v2_152(
x_generated2, num_classes=FLAGS.num_classes, is_training=False)
predicted_labels3 = tf.argmax(end_points3['predictions'], 1)
predicted_logits3 = end_points3['predictions']
disc_var_list3 = slim.get_model_variables()[(len(disc_var_list) +
len(disc_var_list2)):]
# discriminator(resnet v2 101)
with slim.arg_scope(resnet_utils.resnet_arg_scope()):
_, end_points4 = resnet_v2.resnet_v2_101(
x_generated2, num_classes=FLAGS.num_classes, is_training=False)
predicted_labels4 = tf.argmax(end_points4['predictions'], 1)
predicted_logits4 = end_points4['predictions']
disc_var_list4 = slim.get_model_variables()[(len(disc_var_list) +
len(disc_var_list2) +
len(disc_var_list3)):]
# discriminator(inception v4)
with slim.arg_scope(inception.inception_v4_arg_scope()):
_, end_points5 = inception.inception_v4(
x_generated, num_classes=FLAGS.num_classes, is_training=False)
predicted_labels5 = tf.argmax(end_points5['Predictions'], 1)
predicted_logits5 = end_points['Logits']
disc_var_list5 = slim.get_model_variables()[(len(disc_var_list) +
len(disc_var_list2) +
len(disc_var_list3) +
len(disc_var_list4)):]
"""
# discriminator(vgg 19)
with slim.arg_scope(vgg.vgg_arg_scope()):
_, end_points3 = vgg.vgg_19(
x_generated2, num_classes=FLAGS.num_classes, is_training=False)
predicted_labels3 = tf.argmax(end_points3['vgg_19/fc8'], 1);
predicted_logits3 = end_points3['vgg_19/fc8'];
disc_var_list3=slim.get_model_variables()[(len(disc_var_list)+len(disc_var_list2)):];
"""
# loss and optimizer
gen_acc = tf.reduce_mean(
tf.cast(tf.equal(predicted_labels, tf.argmax(y_label, 1)),
tf.float32))
cross_entropy = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(logits=predicted_logits,
labels=y_label))
gen_acc2 = tf.reduce_mean(
tf.cast(tf.equal(predicted_labels2, tf.argmax(y_label, 1)),
tf.float32))
cross_entropy2 = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(logits=predicted_logits2,
labels=y_label))
gen_acc3 = tf.reduce_mean(
tf.cast(tf.equal(predicted_labels3, tf.argmax(y_label, 1)),
tf.float32))
cross_entropy3 = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(logits=predicted_logits3,
labels=y_label))
gen_acc4 = tf.reduce_mean(
tf.cast(tf.equal(predicted_labels4, tf.argmax(y_label, 1)),
tf.float32))
cross_entropy4 = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(logits=predicted_logits4,
labels=y_label))
gen_acc5 = tf.reduce_mean(
tf.cast(tf.equal(predicted_labels5, tf.argmax(y_label, 1)),
tf.float32))
cross_entropy5 = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(logits=predicted_logits5,
labels=y_label))
infi_norm = tf.reduce_mean(
tf.norm(tf.reshape(abs(x_data - x_generated),
[-1, FLAGS.img_size]),
ord=np.inf,
axis=1))
g_loss = -1 * cross_entropy - 1 * cross_entropy2 - 1 * cross_entropy3 - 1 * cross_entropy4 - 1 * cross_entropy5
optimizer = tf.train.AdamOptimizer(0.0001)
g_trainer = optimizer.minimize(g_loss, var_list=g_params)
# get the data and label
img_list = np.sort(glob.glob(FLAGS.input_folder + "*.png"))
total_data = np.zeros(
(len(img_list), FLAGS.img_height, FLAGS.img_width, 3), dtype=float)
for i in range(len(img_list)):
total_data[i] = imread(img_list[i], mode='RGB').astype(
np.float) / 255.0
total_data[i] = total_data[i] * 2.0 - 1.0
# 0~1 -> -1~1
val_data = np.copy(total_data[0])
f = open(FLAGS.label_folder + "true_label", "r")
total_label2 = np.array([i[:-1].split(",")[1] for i in f.readlines()],
dtype=int)
total_label = np.zeros((len(total_data), FLAGS.num_classes), dtype=int)
for i in range(len(total_data)):
total_label[i, total_label2[i]] = 1
val_label = np.copy(total_label[0])
# shuffle
total_idx = range(len(total_data))
np.random.shuffle(total_idx)
total_data = total_data[total_idx]
total_label = total_label[total_idx]
# Run computation
saver = tf.train.Saver(disc_var_list)
saver2 = tf.train.Saver(disc_var_list2)
saver3 = tf.train.Saver(disc_var_list3)
saver4 = tf.train.Saver(disc_var_list4)
saver5 = tf.train.Saver(disc_var_list5)
saver_gen = tf.train.Saver(g_params)
# initialization
init = tf.global_variables_initializer()
with tf.Session() as sess:
sess.run(init)
saver.restore(sess,
FLAGS.checkpoint_path + FLAGS.checkpoint_file_name)
saver2.restore(sess,
FLAGS.checkpoint_path + FLAGS.checkpoint_file_name2)
saver3.restore(sess,
FLAGS.checkpoint_path + FLAGS.checkpoint_file_name3)
saver4.restore(sess,
FLAGS.checkpoint_path + FLAGS.checkpoint_file_name4)
saver5.restore(sess,
FLAGS.checkpoint_path + FLAGS.checkpoint_file_name5)
# training
for i in range(FLAGS.max_epoch):
tr_infi = 0
tr_ce = 0
tr_gen_acc = 0
tr_ce2 = 0
tr_gen_acc2 = 0
tr_ce3 = 0
tr_gen_acc3 = 0
tr_ce4 = 0
tr_gen_acc4 = 0
tr_ce5 = 0
tr_gen_acc5 = 0
for j in range(len(total_data) / FLAGS.batch_size):
batch_data = total_data[j * FLAGS.batch_size:(j + 1) *
FLAGS.batch_size]
batch_label = total_label[j * FLAGS.batch_size:(j + 1) *
FLAGS.batch_size]
acc_p5, ce_p5, acc_p4, ce_p4, acc_p3, ce_p3, acc_p2, ce_p2, acc_p, ce_p, infi_p, _ = sess.run(
[
gen_acc5, cross_entropy5, gen_acc4, cross_entropy4,
gen_acc3, cross_entropy3, gen_acc2, cross_entropy2,
gen_acc, cross_entropy, infi_norm, g_trainer
],
feed_dict={
x_data: batch_data,
y_label: batch_label
})
tr_infi += infi_p
tr_ce += ce_p
tr_gen_acc += acc_p
tr_ce2 += ce_p2
tr_gen_acc2 += acc_p2
tr_ce3 += ce_p3
tr_gen_acc3 += acc_p3
tr_ce4 += ce_p4
tr_gen_acc4 += acc_p4
tr_ce5 += ce_p5
tr_gen_acc5 += acc_p5
print(
str(i + 1) + " Epoch InfiNorm:" + str(tr_infi / (j + 1)) +
",CE: " + str(tr_ce / (j + 1)) + ",Acc: " +
str(tr_gen_acc / (j + 1)) + ",CE2: " + str(tr_ce2 /
(j + 1)) +
",Acc2: " + str(tr_gen_acc2 / (j + 1)) + ",CE3: " +
str(tr_ce3 / (j + 1)) + ",Acc3: " + str(tr_gen_acc3 /
(j + 1)) +
",CE4: " + str(tr_ce4 / (j + 1)) + ",Acc4: " +
str(tr_gen_acc4 / (j + 1)) + ",CE5: " +
str(tr_ce5 / (j + 1)) + ",Acc5: " + str(tr_gen_acc5 /
(j + 1)))
total_idx = range(len(total_data))
np.random.shuffle(total_idx)
total_data = total_data[total_idx]
total_label = total_label[total_idx]
saver_gen.save(
sess, "my-models_iv3_rv250_rv2152_rv2101_iv4/my-model_" +
str(FLAGS.max_epsilon) + ".ckpt")
image = tf.image.decode_jpeg(tf.read_file(image_path), channels=3)
image = tf.image.convert_image_dtype(image, tf.float32)
image = inception_preprocessing.preprocess_image(image,
image_size,
image_size,
is_training=False)
images = tf.expand_dims(image, 0)
if base_network == 'ResNet':
with slim.arg_scope(resnet_v2.resnet_arg_scope(use_batch_norm=True)):
if layers == '50':
net, _ = resnet_v2.resnet_v2_50(images, is_training=False)
elif layers == '101':
net, _ = resnet_v2.resnet_v2_101(images, is_training=False)
elif layers == '152':
net, _ = resnet_v2.resnet_v2_152(images, is_training=False)
else:
with slim.arg_scope(arg_scope):
slim_args = [slim.batch_norm, slim.dropout]
with slim.arg_scope(slim_args, is_training=False):
with tf.variable_scope(base_network, reuse=None) as scope:
if base_network == 'InceptionV3':
net, _ = inception.inception_v3_base(
images, final_endpoint=endpoint, scope=scope)
elif base_network == 'InceptionV3SE':
net, _ = inception.inception_v3_se_base(
images, final_endpoint=endpoint, scope=scope)
elif base_network == 'InceptionV4':
net, _ = inception.inception_v4_base(
images, final_endpoint=endpoint, scope=scope)
elif base_network == 'InceptionResnetV2':
