def graph(x, y, i, x_max, x_min, grad):
eps = 2.0 * FLAGS.max_epsilon / 255.0
num_iter = FLAGS.num_iter
alpha = eps / num_iter
momentum = FLAGS.momentum
num_classes = 1001
# should keep original x here for output
with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
logits_v3, end_points_v3 = inception_v3.inception_v3(
input_diversity(x), num_classes=num_classes, is_training=False)
logits_v3_rotated, _ = inception_v3.inception_v3(
rotate(x), num_classes=num_classes, is_training=False, reuse=True)
with slim.arg_scope(inception_v4.inception_v4_arg_scope()):
logits_v4, end_points_v4 = inception_v4.inception_v4(
input_diversity(x), num_classes=num_classes, is_training=False)
logits_v4_rotated, _ = inception_v4.inception_v4(
rotate(x), num_classes=num_classes, is_training=False, reuse=True)
with slim.arg_scope(inception_resnet_v2.inception_resnet_v2_arg_scope()):
logits_res_v2, end_points_res_v2 = inception_resnet_v2.inception_resnet_v2(
input_diversity(x),
num_classes=num_classes,
is_training=False,
reuse=True)
logits_res_v2_rotated, _ = inception_resnet_v2.inception_resnet_v2(
rotate(x), num_classes=num_classes, is_training=False, reuse=True)
with slim.arg_scope(resnet_v2.resnet_arg_scope()):
logits_resnet, end_points_resnet = resnet_v2.resnet_v2_152(
input_diversity(x), num_classes=num_classes, is_training=False)
logits_resnet_rotated, _ = resnet_v2.resnet_v2_152(
rotate(x), num_classes=num_classes, is_training=False, reuse=True)
logits = (logits_v3 + logits_v4 + logits_res_v2 + logits_resnet +
logits_v3_rotated + logits_v4_rotated + logits_res_v2_rotated +
logits_resnet_rotated) / 8
auxlogits = (end_points_v3['AuxLogits'] + end_points_v4['AuxLogits'] +
end_points_res_v2['AuxLogits']) / 3
cross_entropy = tf.losses.softmax_cross_entropy(y,
logits,
label_smoothing=0.0,
weights=1.0)
cross_entropy += tf.losses.softmax_cross_entropy(y,
auxlogits,
label_smoothing=0.0,
weights=0.4)
noise = tf.gradients(cross_entropy, x)[0]
# noise = tf.nn.depthwise_conv2d(noise, stack_kernel, strides=[1, 1, 1, 1], padding='SAME')
noise = noise / tf.reduce_mean(tf.abs(noise), [1, 2, 3], keep_dims=True)
noise = momentum * grad + noise
x = x + alpha * tf.sign(noise)
x = tf.clip_by_value(x, x_min, x_max)
i = tf.add(i, 1)
return x, y, i, x_max, x_min, noise
def testVariablesSetDevice(self):
batch_size = 5
height, width = 299, 299
num_classes = 1000
inputs = tf.random_uniform((batch_size, height, width, 3))
# Force all Variables to reside on the device.
with tf.variable_scope('on_cpu'), tf.device('/cpu:0'):
inception_v4.inception_v4(inputs, num_classes)
with tf.variable_scope('on_gpu'), tf.device('/gpu:0'):
inception_v4.inception_v4(inputs, num_classes)
for v in tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES,
scope='on_cpu'):
self.assertDeviceEqual(v.device, '/cpu:0')
for v in tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES,
scope='on_gpu'):
self.assertDeviceEqual(v.device, '/gpu:0')
def __init__(self, ckpt_path):
checkpoints_dir = ckpt_path
self.batch_size = 10
# classify initial
inception_v4_arg_scope = inception_utils.inception_arg_scope()
self.image_size = 299
arg_scope = inception_utils.inception_arg_scope()
number_classes = 1001
self.input_batch = tf.placeholder(
tf.float32, shape=[None, self.image_size, self.image_size, 3])
with slim.arg_scope(arg_scope):
logits, end_points = inception_v4.inception_v4(
self.input_batch,
num_classes=number_classes,
is_training=False)
weights_restored_from_file = slim.get_variables_to_restore(
exclude=['InceptionV4/Logits'])
init_fn = slim.assign_from_checkpoint_fn(
os.path.join(checkpoints_dir, 'inception_v4.ckpt'),
weights_restored_from_file,
ignore_missing_vars=True) # assign varible
self.feature_global = end_points['global_pool']
self.sess = tf.Session()
init_fn(self.sess)
self.dist = DistanceBuilder()
def main(argv=None):
#加载预处理好的数据
#定义inception_v4的输入,
init_data()
images = tf.placeholder(dtype=tf.float32, shape=[FLAGS.batch_size, FLAGS.image_size, FLAGS.image_size, 3])
multi_hot_labels = tf.placeholder(dtype=tf.float32, shape=[FLAGS.batch_size, FLAGS.num_classes])
# labels = tf.one_hot(labels, depth=5)
#定义inception_v4模型，因为谷歌给出的只有模型参数取值，所以这里需要在这个代码中定义inception_v4的模型结构。虽然理论上需要区分训练和
#测试中使用的模型，也就是说在测试时应该使用is_training=False, 但是因为预预先训练好的inception-v3模型中使用的batch normalization参数
#与新的数据会有差异，导致结果很差，所以这里直接使用同一个模型来进行测试
global_step = tf.Variable(0, trainable=False)
with slim.arg_scope(inception_v4.inception_v4_arg_scope()):
logits, _ = inception_v4.inception_v4(images, num_classes=FLAGS.num_classes, dropout_keep_prob=1.0, is_training=False, create_aux_logits=False)
#获取需要训练的变量
final_tensor = tf.nn.sigmoid(logits, name=FLAGS.final_tensor_name)
#定义保存新的训练好的模型函数
saver = tf.train.Saver()
with tf.Session() as sess:
#初始化没有加载进来的变量。注意这个过程一定要在模型加载之前，否则初始化过程会将已经加载好的变量重新加载
tf.global_variables_initializer().run()
tf.local_variables_initializer().run()
saver.restore(sess, FLAGS.ckpt_save_file_path)
print("restored global_step:",global_step.eval())
predict_array = []
for step in range(total_steps):
images_val, image_paths_val = get_image_data_array(step+1)
# print(sess.graph.get_tensor_by_name("image_batch:0").eval())
final_tensor_val = sess.run(final_tensor, feed_dict={images:images_val})
for i in range(len(final_tensor_val)):
sorted_array = final_tensor_val[i].argsort()[::-1]
prediction_result=[LABEL_DICT[label_index+1]+"("+str(label_index+1)+"):"+str(final_tensor_val[i][label_index]) for label_index in sorted_array[:FLAGS.print_count]]
predict_array.append(image_paths_val[i]+" "+str(prediction_result))
for i in range(len(all_image_paths)):
print(predict_array[i])
def graph(x, y, i, x_max, x_min, grad):
x = x + grad
eps = 2.0 * FLAGS.max_epsilon / 255.0
num_iter = FLAGS.num_iter
alpha = eps / num_iter
momentum = FLAGS.momentum
num_classes = 1001
with slim.arg_scope(
inception_v4.inception_v4_arg_scope()): # 作用是给list_ops中的内容设置默认值
logits_v4, end_points_v4 = inception_v4.inception_v4(
x + grad * momentum, num_classes=num_classes, is_training=False)
pred = tf.argmax(end_points_v4['Predictions'], 1)
first_round = tf.cast(tf.equal(i, 0), tf.int64)
y = first_round * pred + (1 - first_round) * y
one_hot = tf.one_hot(y, num_classes)
logits = (logits_v4) / 7.25
auxlogits = (end_points_v4['AuxLogits']) / 6.25
cross_entropy = tf.losses.softmax_cross_entropy(one_hot,
logits,
label_smoothing=0.0,
weights=1.0)
cross_entropy += tf.losses.softmax_cross_entropy(one_hot,
auxlogits,
label_smoothing=0.0,
weights=0.4)
noise = tf.gradients(cross_entropy, x)[0]
noise = noise / tf.reduce_mean(tf.abs(noise), [1, 2, 3], keep_dims=True)
noise = momentum * grad + noise
x = x + alpha * tf.sign(noise)
x = tf.clip_by_value(x, x_min, x_max)
i = tf.add(i, 1)
return x, y, i, x_max, x_min, noise
def __init__(self):
self.slim = tf.contrib.slim
self.image_size = inception_v4.inception_v4.default_image_size
self.checkpoints_dir = 'checkpoints'
self.names = imagenet.create_readable_names_for_imagenet_labels()
self.arg_scope = inception_v4.inception_v4_arg_scope()
self.image = tf.placeholder(tf.uint8, [480, 640, 3])
self.processed_image = inception_preprocessing.preprocess_image(self.image,
self.image_size, self.image_size,
is_training=False)
self.processed_images = tf.expand_dims(self.processed_image, 0)
# processed_images will be a 1x299x299x3 tensor of float32
# Create the model, use the default arg scope to configure the batch norm parameters.
with self.slim.arg_scope(self.arg_scope):
self.logits, self.end_points = inception_v4.inception_v4(self.processed_images, num_classes=1001,
is_training=False)
self.probs = tf.nn.softmax(self.logits)
self.init_fn = self.slim.assign_from_checkpoint_fn(
os.path.join(self.checkpoints_dir, 'inception_v4.ckpt'),
self.slim.get_model_variables('InceptionV4'))
config = tf.compat.v1.ConfigProto()
config.gpu_options.allow_growth = True
self.session = tf.compat.v1.Session(config=config)
self.init_fn(self.session)
def test_inception_v4(img_dir):
"""
Test Inception-V4 with a single image.
:param img_dir: Path of the image to be classified
:return: classification result and probability of a single image
"""
img = cv2.imread(img_dir)
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
img = cv2.resize(img, (299, 299)) / 255
img = img.reshape((1, 299, 299, 3))
tf.reset_default_graph()
inputs = tf.placeholder(name='input_images',
shape=[None, 299, 299, 3],
dtype=tf.float32)
with slim.arg_scope(inception_v4_arg_scope()):
_, _ = inception_v4(inputs, 1001, is_training=False)
with tf.Session() as sess:
tf.train.Saver().restore(sess, './models/inception_v4.ckpt')
inputs = sess.graph.get_tensor_by_name('input_images:0')
outputs = sess.graph.get_tensor_by_name(
'InceptionV4/Logits/Predictions:0')
pred = tf.argmax(outputs, axis=1)[0]
prob = tf.reduce_max(outputs, axis=1)[0]
pred, prob = sess.run([pred, prob], feed_dict={inputs: img})
name = label_dict[pred]
print('Result of Inception-V4:', name, prob)
return name, prob
def graph(x, y, i, x_max, x_min, grad):
eps = 2.0 * FLAGS.max_epsilon / 255.0
num_iter = FLAGS.num_iter
alpha = eps / num_iter
momentum = FLAGS.momentum
num_classes = 1001
with slim.arg_scope(inception_v4.inception_v4_arg_scope()):
logits_v4, end_points_v4 = inception_v4.inception_v4(
x, num_classes=num_classes, is_training=False)
pred = tf.argmax(end_points_v4['Predictions'], 1)
first_round = tf.cast(tf.equal(i, 0), tf.int64)
y = first_round * pred + (1 - first_round) * y
one_hot = tf.one_hot(y, num_classes)
logits = logits_v4
auxlogits = end_points_v4['AuxLogits']
cross_entropy = tf.losses.softmax_cross_entropy(one_hot,
logits,
label_smoothing=0.0,
weights=1.0)
cross_entropy += tf.losses.softmax_cross_entropy(one_hot,
auxlogits,
label_smoothing=0.0,
weights=0.4)
noise = tf.gradients(cross_entropy, x)[0]
x = x + alpha * tf.sign(noise)
x = tf.clip_by_value(x, x_min, x_max)
i = tf.add(i, 1)
return x, y, i, x_max, x_min, noise
def main(argv=None):
#加载预处理好的数据
#定义inception_v3的输入,
images, labels, multi_hot_labels = get_image_batch()
# labels = tf.one_hot(labels, depth=5)
#定义inception_v3模型，因为谷歌给出的只有模型参数取值，所以这里需要在这个代码中定义inception_v3的模型结构。虽然理论上需要区分训练和
#测试中使用的模型，也就是说在测试时应该使用is_training=False, 但是因为预预先训练好的inception-v3模型中使用的batch normalization参数
#与新的数据会有差异，导致结果很差，所以这里直接使用同一个模型来进行测试
global_step = tf.Variable(0, trainable=False)
with slim.arg_scope(inception_v4.inception_v4_arg_scope()):
logits, _ = inception_v4.inception_v4(images,
num_classes=FLAGS.num_classes,
dropout_keep_prob=1.0)
#获取需要训练的变量
final_tensor = tf.nn.sigmoid(logits, name=FLAGS.final_tensor_name)
tf.summary.histogram(FLAGS.final_tensor_name + '/activations',
final_tensor)
#定义交叉熵损失，注意在模型定义的时候已经将正则化损失加入损失集合了
tf.losses.sigmoid_cross_entropy(multi_class_labels=multi_hot_labels,
logits=logits,
weights=1.0)
#定义训练过程。这里minimize的过程中指定了需要优化的变量集合
# learning_rate = LEARNING_RATE_BASE
learning_rate = tf.train.exponential_decay(
FLAGS.learning_rate_base,
global_step=global_step,
decay_steps=FLAGS.learning_rate_decay_steps,
decay_rate=FLAGS.learning_rate_decay)
loss = tf.losses.get_total_loss()
train_step = tf.train.RMSPropOptimizer(learning_rate).minimize(
loss, global_step=global_step)
#计算正确率
with tf.name_scope('evaluation'):
correct_prediction = tf.equal(tf.round(final_tensor), multi_hot_labels)
evaluation_step = tf.reduce_mean(tf.cast(correct_prediction,
tf.float32),
name='accuracy')
#定义保存新的训练好的模型函数
saver = tf.train.Saver()
with tf.Session() as sess:
#初始化没有加载进来的变量。注意这个过程一定要在模型加载之前，否则初始化过程会将已经加载好的变量重新加载
tf.global_variables_initializer().run()
tf.local_variables_initializer().run()
# ckpt = tf.train.get_checkpoint_state(FLAGS.ckpt_save_dir)
# if ckpt and ckpt.model_checkpoint_path:
saver.restore(sess=sess, save_path=FLAGS.ckpt_save_file_path)
print("restored global_step:", global_step.eval())
output_graph_def = graph_util.convert_variables_to_constants(
sess, sess.graph.as_graph_def(), [
FLAGS.image_batch_name, FLAGS.final_tensor_name,
FLAGS.accuracy_name
])
with gfile.FastGFile(FLAGS.output_graph, 'wb') as f:
f.write(output_graph_def.SerializeToString())
print("export %s over!" % (FLAGS.output_graph))
def add_inference_node(self, is_training=True):
with slim.arg_scope(inception_v4.inception_v4_arg_scope()):
self.output, self.end_points = inception_v4.inception_v4(
self.input,
num_classes=trafficlight_datasets.NUM_CLASSES,
is_training=is_training,
dropout_keep_prob=0.8,
create_aux_logits=True)
return
def inference_inception_v4(x_input, dropout_keep_prob=1, num_classes=1001):
with slim.arg_scope(inception_v4.inception_v4_arg_scope()):
logits, _ = inception_v4.inception_v4(x_input,
dropout_keep_prob=dropout_keep_prob, create_aux_logits=False,
num_classes=num_classes, is_training=False)
probs = tf.nn.softmax(logits)
model_vars = [var for var in tf.global_variables() \
if var.name.startswith('InceptionV4/')]
return probs, logits, model_vars
def graph(x, y, i, x_max, x_min, grad, grad2):
eps = 2.0 * FLAGS.max_epsilon / 255.0
num_classes = 1001
with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
logits_v3, end_points_v3 = inception_v3.inception_v3(
input_diversity(x),
num_classes=num_classes,
is_training=False,
reuse=tf.AUTO_REUSE)
with slim.arg_scope(inception_v4.inception_v4_arg_scope()):
logits_v4, end_points_v4 = inception_v4.inception_v4(
input_diversity(x),
num_classes=num_classes,
is_training=False,
reuse=tf.AUTO_REUSE)
with slim.arg_scope(inception_resnet_v2.inception_resnet_v2_arg_scope()):
logits_res_v2, end_points_res_v2 = inception_resnet_v2.inception_resnet_v2(
input_diversity(x),
num_classes=num_classes,
is_training=False,
reuse=tf.AUTO_REUSE)
with slim.arg_scope(resnet_v2.resnet_arg_scope()):
logits_resnet, end_points_resnet = resnet_v2.resnet_v2_101(
input_diversity(x),
num_classes=num_classes,
is_training=False,
scope='resnet_v2_101',
reuse=tf.AUTO_REUSE)
logits = (logits_v3 + logits_v4 + logits_res_v2 + logits_resnet) / 4
auxlogits = (end_points_v3['AuxLogits'] + end_points_v4['AuxLogits'] +
end_points_res_v2['AuxLogits']) / 3
cross_entropy = tf.losses.softmax_cross_entropy(y,
logits,
label_smoothing=0.0,
weights=1.0)
cross_entropy += tf.losses.softmax_cross_entropy(y,
auxlogits,
label_smoothing=0.0,
weights=0.4)
noise = tf.gradients(cross_entropy, x)[0]
noise = tf.nn.depthwise_conv2d(noise,
stack_kernel,
strides=[1, 1, 1, 1],
padding='SAME')
noise1 = grad + 1.5 * noise
noise2 = grad2 + 1.9 * noise * noise
x = x + (eps / 17.6786) * (
(1 - 0.9**(i + 1)) / tf.sqrt(1 - 0.99**(i + 1))) * tf.tanh(
1.3 * noise1 / tf.sqrt(noise2))
x = tf.clip_by_value(x, x_min, x_max)
i = tf.add(i, 1)
return x, y, i, x_max, x_min, noise1, noise2
def choose_model(x, model):
"""
选择模型
:param x:
:param model:
:return:
"""
# 模型保存路径，模型名，预训练文件路径，前向传播
if model == 'Alex':
log_dir = "E:/alum/log/Alex"
y, _ = alexnet.alexnet_v2(
x,
num_classes=CLASSES, # 分类的类别
is_training=True, # 是否在训练
dropout_keep_prob=1.0, # 保留比率
spatial_squeeze=True, # 压缩掉1维的维度
global_pool=GLOBAL_POOL) # 输入不是规定的尺寸时，需要global_pool
elif model == 'VGG':
log_dir = "E:/alum/log/VGG"
y, _ = vgg.vgg_16(x,
num_classes=CLASSES,
is_training=True,
dropout_keep_prob=1.0,
spatial_squeeze=True,
global_pool=GLOBAL_POOL)
elif model == 'VGG2':
log_dir = "E:/alum/log/VGG2"
y, _ = vgg.vgg_16(x,
num_classes=CLASSES,
is_training=True,
dropout_keep_prob=1.0,
spatial_squeeze=True,
global_pool=GLOBAL_POOL)
elif model == 'Incep4':
log_dir = "E:/alum/log/Incep4"
y, _ = inception_v4.inception_v4(x,
num_classes=CLASSES,
is_training=True,
dropout_keep_prob=1.0,
reuse=None,
scope='InceptionV4',
create_aux_logits=True)
elif model == 'Res':
log_dir = "E:/alum/log/Res"
y, _ = resnet_v2.resnet_v2_50(x,
num_classes=CLASSES,
is_training=True,
global_pool=GLOBAL_POOL,
output_stride=None,
spatial_squeeze=True,
reuse=None,
scope='resnet_v2_50')
else:
print('Error: model name not exist')
return
return y, log_dir
def testTrainEvalWithReuse(self):
train_batch_size = 5
eval_batch_size = 2
height, width = 150, 150
num_classes = 1000
with self.test_session() as sess:
train_inputs = tf.random_uniform(
(train_batch_size, height, width, 3))
inception_v4.inception_v4(train_inputs, num_classes)
eval_inputs = tf.random_uniform(
(eval_batch_size, height, width, 3))
logits, _ = inception_v4.inception_v4(eval_inputs,
num_classes,
is_training=False,
reuse=True)
predictions = tf.argmax(logits, 1)
sess.run(tf.global_variables_initializer())
output = sess.run(predictions)
self.assertEquals(output.shape, (eval_batch_size, ))
def graph(x, y, i, grad):
num_classes = 1001
x = x
with slim.arg_scope(inception_v4.inception_v4_arg_scope()):
logits_v4, end_points_v4 = inception_v4.inception_v4(
x, num_classes=num_classes, is_training=False)
pred = tf.argmax(end_points_v4['Predictions'], 1)
first_round = tf.cast(tf.equal(i, 0), tf.int64)
y = first_round * pred + (1 - first_round) * y
return x, y, i, grad
def testBuildPreLogitsNetwork(self):
batch_size = 5
height, width = 299, 299
num_classes = None
inputs = tf.random_uniform((batch_size, height, width, 3))
net, end_points = inception_v4.inception_v4(inputs, num_classes)
self.assertTrue(net.op.name.startswith('InceptionV4/Logits/AvgPool'))
self.assertListEqual(net.get_shape().as_list(),
[batch_size, 1, 1, 1536])
self.assertFalse('Logits' in end_points)
self.assertFalse('Predictions' in end_points)
def testBuildWithoutAuxLogits(self):
batch_size = 5
height, width = 299, 299
num_classes = 1000
inputs = tf.random_uniform((batch_size, height, width, 3))
logits, endpoints = inception_v4.inception_v4(inputs,
num_classes,
create_aux_logits=False)
self.assertFalse('AuxLogits' in endpoints)
self.assertTrue(logits.op.name.startswith('InceptionV4/Logits'))
self.assertListEqual(logits.get_shape().as_list(),
[batch_size, num_classes])
def discrimination_score(inputs,
num_classes,
is_training=True,
reuse=tf.AUTO_REUSE,
scope='InceptionV4'):
"""
Raw View Descriptor Generation
first part of the network (FCN) to get the raw descriptor in the view level.
The “FCN” part is the top five convolutional layers of GoogLeNet.
(mid-level representation)
Extract the raw view descriptors.
Compared with deeper CNN, shallow FCN could have more position information,
which is needed for the followed grouping module and the deeper CNN will have
the content information which could represent the view feature better.
Args:
inputs: N x V x H x W x C tensor
scope:
"""
raw_view_descriptors = []
final_view_descriptors = []
view_discrimination_scores = []
n_views = inputs.get_shape().as_list()[1]
# transpose views: (NxVxHxWxC) -> (VxNxHxWxC)
views = tf.transpose(inputs, perm=[1, 0, 2, 3, 4])
for index in range(n_views):
batch_view = tf.gather(views, index) # N x H x W x C
with slim.arg_scope(inception_v4.inception_v4_arg_scope()):
raw_desc, net, end_points = \
inception_v4.inception_v4(batch_view, v_scope='_view' + str(index),
is_training=is_training, reuse=reuse, scope=scope)
raw_view_descriptors.append(raw_desc['raw_desc'])
final_view_descriptors.append(net)
# GAP layer to obtain the discrimination scores from raw view descriptors.
raw = tf.reduce_mean(raw_desc['raw_desc'], [1, 2], keepdims=True)
raw = slim.conv2d(raw, num_classes, [1, 1], activation_fn=None)
raw = tf.reduce_max(raw, axis=[1, 2, 3])
batch_view_score = tf.nn.sigmoid(tf.log(tf.abs(raw)))
view_discrimination_scores.append(batch_view_score)
# # Print name and shape of parameter nodes (values not yet initialized)
# tf.logging.info("++++++++++++++++++++++++++++++++++")
# tf.logging.info("Parameters")
# tf.logging.info("++++++++++++++++++++++++++++++++++")
# for v in slim.get_model_variables():
# tf.logging.info('name = %s, shape = %s' % (v.name, v.get_shape()))
return view_discrimination_scores, raw_view_descriptors, final_view_descriptors
def testGlobalPool(self):
batch_size = 2
height, width = 400, 600
num_classes = 1000
inputs = tf.random_uniform((batch_size, height, width, 3))
logits, end_points = inception_v4.inception_v4(inputs, num_classes)
self.assertTrue(logits.op.name.startswith('InceptionV4/Logits'))
self.assertListEqual(logits.get_shape().as_list(),
[batch_size, num_classes])
pre_pool = end_points['Mixed_7d']
self.assertListEqual(pre_pool.get_shape().as_list(),
[batch_size, 11, 17, 1536])
def graph(x, y, i, x_max, x_min, grad):
eps = 2.0 * FLAGS.max_epsilon / 255.0
num_iter = FLAGS.num_iter
alpha = eps / num_iter
momentum = FLAGS.momentum
num_classes = 1001
with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
logits_v3, end_points_v3 = inception_v3.inception_v3(
input_diversity(x), num_classes=num_classes, is_training=False)
with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
logits_adv_v3, end_points_adv_v3 = inception_v3.inception_v3(
input_diversity(x), num_classes=num_classes, is_training=False, scope='AdvInceptionV3')
with slim.arg_scope(inception_v4.inception_v4_arg_scope()):
logits_v4, end_points_v4 = inception_v4.inception_v4(
input_diversity(x), num_classes=num_classes, is_training=False)
with slim.arg_scope(inception_resnet_v2.inception_resnet_v2_arg_scope()):
logits_res_v2, end_points_res_v2 = inception_resnet_v2.inception_resnet_v2(
input_diversity(x), num_classes=num_classes, is_training=False)
with slim.arg_scope(inception_resnet_v2.inception_resnet_v2_arg_scope()):
logits_ensadv_res_v2, end_points_ensadv_res_v2 = inception_resnet_v2.inception_resnet_v2(
input_diversity(x), num_classes=num_classes, is_training=False, scope='EnsAdvInceptionResnetV2')
with slim.arg_scope(resnet_v2.resnet_arg_scope()):
logits_resnet, end_points_resnet = resnet_v2.resnet_v2_50(
input_diversity(x), num_classes=num_classes, is_training=False)
logits = (logits_v3 + 0.25 * logits_adv_v3 + logits_v4 + \
logits_res_v2 + logits_ensadv_res_v2 + logits_resnet) / 5.25
auxlogits = (end_points_v3['AuxLogits'] + 0.25 * end_points_adv_v3['AuxLogits'] + end_points_v4['AuxLogits'] + \
end_points_res_v2['AuxLogits'] + end_points_ensadv_res_v2['AuxLogits']) / 4.25
cross_entropy = tf.losses.softmax_cross_entropy(y,
logits,
label_smoothing=0.0,
weights=1.0)
cross_entropy += tf.losses.softmax_cross_entropy(y,
auxlogits,
label_smoothing=0.0,
weights=0.4)
noise = tf.gradients(cross_entropy, x)[0]
noise = tf.nn.depthwise_conv2d(noise, stack_kernel, strides=[1, 1, 1, 1], padding='SAME')
noise = noise / tf.reshape(tf.contrib.keras.backend.std(tf.reshape(noise, [FLAGS.batch_size, -1]), axis=1), [FLAGS.batch_size, 1, 1, 1])
noise = momentum * grad + noise
noise = noise / tf.reshape(tf.contrib.keras.backend.std(tf.reshape(noise, [FLAGS.batch_size, -1]), axis=1), [FLAGS.batch_size, 1, 1, 1])
x = x - alpha * tf.clip_by_value(tf.round(noise), -2, 2)
x = tf.clip_by_value(x, x_min, x_max)
i = tf.add(i, 1)
return x, y, i, x_max, x_min, noise
def endpoints(image, is_training):
if image.get_shape().ndims != 4:
raise ValueError('Input must be of size [batch, height, width, 3]')
image = image - tf.constant(_RGB_MEAN, dtype=tf.float32, shape=(1,1,1,3))
with tf.contrib.slim.arg_scope(inception_arg_scope(batch_norm_decay=0.9, weight_decay=0.0002)):
_, endpoints = inception_v4(image, num_classes=None, is_training=is_training)
endpoints['model_output'] = endpoints['global_pool'] = tf.reduce_mean(
endpoints['Mixed_7d'], [1, 2], name='pool5', keep_dims=False)
return endpoints, 'InceptionV4'
def testEvaluation(self):
batch_size = 2
height, width = 299, 299
num_classes = 1000
with self.test_session() as sess:
eval_inputs = tf.random_uniform((batch_size, height, width, 3))
logits, _ = inception_v4.inception_v4(eval_inputs,
num_classes,
is_training=False)
predictions = tf.argmax(logits, 1)
sess.run(tf.global_variables_initializer())
output = sess.run(predictions)
self.assertEquals(output.shape, (batch_size, ))
def testUnknownBatchSize(self):
batch_size = 1
height, width = 299, 299
num_classes = 1000
with self.test_session() as sess:
inputs = tf.placeholder(tf.float32, (None, height, width, 3))
logits, _ = inception_v4.inception_v4(inputs, num_classes)
self.assertTrue(logits.op.name.startswith('InceptionV4/Logits'))
self.assertListEqual(logits.get_shape().as_list(),
[None, num_classes])
images = tf.random_uniform((batch_size, height, width, 3))
sess.run(tf.global_variables_initializer())
output = sess.run(logits, {inputs: images.eval()})
self.assertEquals(output.shape, (batch_size, num_classes))
def _build(self, x_input=None):
reuse = True if self.built else None
if x_input is None:
x_input = self.input
with slim.arg_scope(inception_v4.inception_v4_arg_scope()):
logits, end_points = inception_v4.inception_v4(
x_input, num_classes=self.num_classes, is_training=False,
reuse=reuse)
self.built = True
self.end_points = end_points
self.logits = logits
if not self.ckpt_loaded:
saver = tf.train.Saver(slim.get_model_variables())
saver.restore(self.sess, ckpt_dir + 'inception_v4.ckpt')
self.ckpt_loaded = True
def testAllEndPointsShapes(self):
batch_size = 5
height, width = 299, 299
num_classes = 1000
inputs = tf.random_uniform((batch_size, height, width, 3))
_, end_points = inception_v4.inception_v4(inputs, num_classes)
endpoints_shapes = {
'Conv2d_1a_3x3': [batch_size, 149, 149, 32],
'Conv2d_2a_3x3': [batch_size, 147, 147, 32],
'Conv2d_2b_3x3': [batch_size, 147, 147, 64],
'Mixed_3a': [batch_size, 73, 73, 160],
'Mixed_4a': [batch_size, 71, 71, 192],
'Mixed_5a': [batch_size, 35, 35, 384],
# 4 x Inception-A blocks
'Mixed_5b': [batch_size, 35, 35, 384],
'Mixed_5c': [batch_size, 35, 35, 384],
'Mixed_5d': [batch_size, 35, 35, 384],
'Mixed_5e': [batch_size, 35, 35, 384],
# Reduction-A block
'Mixed_6a': [batch_size, 17, 17, 1024],
# 7 x Inception-B blocks
'Mixed_6b': [batch_size, 17, 17, 1024],
'Mixed_6c': [batch_size, 17, 17, 1024],
'Mixed_6d': [batch_size, 17, 17, 1024],
'Mixed_6e': [batch_size, 17, 17, 1024],
'Mixed_6f': [batch_size, 17, 17, 1024],
'Mixed_6g': [batch_size, 17, 17, 1024],
'Mixed_6h': [batch_size, 17, 17, 1024],
# Reduction-A block
'Mixed_7a': [batch_size, 8, 8, 1536],
# 3 x Inception-C blocks
'Mixed_7b': [batch_size, 8, 8, 1536],
'Mixed_7c': [batch_size, 8, 8, 1536],
'Mixed_7d': [batch_size, 8, 8, 1536],
# Logits and predictions
'AuxLogits': [batch_size, num_classes],
'global_pool': [batch_size, 1, 1, 1536],
'PreLogitsFlatten': [batch_size, 1536],
'Logits': [batch_size, num_classes],
'Predictions': [batch_size, num_classes]
}
self.assertItemsEqual(endpoints_shapes.keys(), end_points.keys())
for endpoint_name in endpoints_shapes:
expected_shape = endpoints_shapes[endpoint_name]
self.assertTrue(endpoint_name in end_points)
self.assertListEqual(
end_points[endpoint_name].get_shape().as_list(),
expected_shape)
def graph(x, y, i, x_max, x_min, grad, eg):
eps = 2.0 * FLAGS.max_epsilon / 255.0
num_iter = FLAGS.num_iter
alpha = eps / num_iter
num_classes = 1001
ro = 0.9
beta = 0.89
v = 0.1
eg = ro * eg + (1 - ro) * tf.square(grad)
rms = tf.sqrt(eg + 0.000000001)
x_n = x + (alpha / rms)*grad
with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
logits_v3, end_points_v3 = inception_v3.inception_v3(
input_diversity(x_n), num_classes=num_classes, is_training=False, reuse=tf.AUTO_REUSE)
with slim.arg_scope(inception_v4.inception_v4_arg_scope()):
logits_v4, end_points_v4 = inception_v4.inception_v4(
input_diversity(x_n), num_classes=num_classes, is_training=False, reuse=tf.AUTO_REUSE)
with slim.arg_scope(inception_resnet_v2.inception_resnet_v2_arg_scope()):
logits_res_v2, end_points_res_v2 = inception_resnet_v2.inception_resnet_v2(
input_diversity(x_n), num_classes=num_classes, is_training=False, reuse=tf.AUTO_REUSE)
with slim.arg_scope(resnet_v2.resnet_arg_scope()):
logits_resnet, end_points_resnet = resnet_v2.resnet_v2_101(
input_diversity(x_n), num_classes=num_classes, is_training=False, scope='resnet_v2_101', reuse=tf.AUTO_REUSE)
logits = (logits_v3 + logits_v4 + logits_res_v2 + logits_resnet) / 4
auxlogits = (end_points_v3['AuxLogits'] + end_points_v4['AuxLogits'] + end_points_res_v2['AuxLogits']) / 3
cross_entropy = tf.losses.softmax_cross_entropy(y,
logits,
label_smoothing=0.0,
weights=1.0)
cross_entropy += tf.losses.softmax_cross_entropy(y,
auxlogits,
label_smoothing=0.0,
weights=0.4)
noise = tf.gradients(cross_entropy, x_n)[0]
noise = tf.nn.depthwise_conv2d(noise, stack_kernel, strides=[1, 1, 1, 1], padding='SAME')
noise1 = noise / tf.reduce_mean(tf.abs(noise), [1, 2, 3], keep_dims=True)
noise = beta * grad + (1-beta) * noise1
noise2 = (1-v) * noise + v * noise1
x = x + alpha * tf.sign(noise2)
x = tf.clip_by_value(x, x_min, x_max)
i = tf.add(i, 1)
return x, y, i, x_max, x_min, noise, eg
def graph(x, adv, y, t_y, i, x_max, x_min, grad, amplification):
target_one_hot = tf.one_hot(t_y, 1001)
true_one_hot = tf.one_hot(y, 1001)
eps = 2.0 * FLAGS.max_epsilon / 255.0
num_iter = FLAGS.num_iter
alpha = eps / num_iter
alpha_beta = alpha
gamma = alpha_beta
momentum = FLAGS.momentum
num_classes = 1001
# with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
# logits_v3, end_points_v3 = inception_v3.inception_v3(
# adv, num_classes=num_classes, is_training=False, reuse = True)
# auxlogit_v3 = end_points_v3['AuxLogits']
with slim.arg_scope(inception_v4.inception_v4_arg_scope()):
logits_v4, end_points_v4 = inception_v4.inception_v4(
adv, num_classes=num_classes, is_training=False, reuse=True)
auxlogit_v4 = end_points_v4['AuxLogits']
with slim.arg_scope(resnet_v2.resnet_arg_scope()):
logits_resnet_152, end_points_resnet = resnet_v2.resnet_v2_152(
adv, num_classes=num_classes, is_training=False, reuse=True)
with slim.arg_scope(inception_resnet_v2.inception_resnet_v2_arg_scope()):
logits_Incres, end_points_IR = inception_resnet_v2.inception_resnet_v2(
adv, num_classes=num_classes, is_training=False, reuse=True)
auxlogit_IR = end_points_IR['AuxLogits']
logits = (logits_Incres + logits_resnet_152 + logits_v4) / 3.0
auxlogit = (auxlogit_IR + auxlogit_v4) / 2.0
target_cross_entropy = tf.losses.softmax_cross_entropy(target_one_hot,
logits,
label_smoothing=0.0,
weights=1.0)
target_cross_entropy += tf.losses.softmax_cross_entropy(
target_one_hot, auxlogit, label_smoothing=0.0, weights=1.0)
noise = tf.gradients(target_cross_entropy, adv)[0]
adv = adv - alpha * n_staircase_sign(noise, num_of_K)
adv = tf.clip_by_value(adv, x_min, x_max)
i = tf.add(i, 1)
return x, adv, y, t_y, i, x_max, x_min, noise, amplification
def output_features(image_batch):
if FLAGS.model == 'inceptionv4':
with slim.arg_scope(inception_v4.inception_v4_arg_scope()):
net, _ = inception_v4.inception_v4(image_batch,
None,
is_training=False)
net = tf.squeeze(net, [1, 2])
elif FLAGS.model == 'resnet101v2':
with slim.arg_scope(resnet_v2.resnet_arg_scope()):
net, _ = resnet_v2.resnet_v2_101(image_batch,
None,
is_training=False,
global_pool=True)
net = tf.squeeze(net, [1, 2])
else:
raise KeyError('{} is not supported'.format(FLAGS.model))
return net
def target_model(x):
num_classes=1001
with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
logits_v3, end_points_v3 = inception_v3.inception_v3(
input_diversity(x), num_classes=num_classes, is_training=False)
with slim.arg_scope(inception_v4.inception_v4_arg_scope()):
logits_v4, end_points_v4 = inception_v4.inception_v4(
input_diversity(x), num_classes=num_classes, is_training=False)
with slim.arg_scope(inception_resnet_v2.inception_resnet_v2_arg_scope()):
logits_res_v2, end_points_res_v2 = inception_resnet_v2.inception_resnet_v2(
input_diversity(x), num_classes=num_classes, is_training=False, reuse=True)
with slim.arg_scope(resnet_v2.resnet_arg_scope()):
logits_resnet, end_points_resnet = resnet_v2.resnet_v2_152(
input_diversity(x), num_classes=num_classes, is_training=False)
with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
logits_adv_v3, end_points_adv_v3 = inception_v3.inception_v3(
input_diversity(x), num_classes=num_classes, is_training=False, scope='AdvInceptionV3')
with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
logits_ens3_adv_v3, end_points_ens3_adv_v3 = inception_v3.inception_v3(
input_diversity(x), num_classes=num_classes, is_training=False, scope='Ens3AdvInceptionV3')
with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
logits_ens4_adv_v3, end_points_ens4_adv_v3 = inception_v3.inception_v3(
input_diversity(x), num_classes=num_classes, is_training=False, scope='Ens4AdvInceptionV3')
with slim.arg_scope(inception_resnet_v2.inception_resnet_v2_arg_scope()):
logits_ensadv_res_v2, end_points_ensadv_res_v2 = inception_resnet_v2.inception_resnet_v2(
input_diversity(x), num_classes=num_classes, is_training=False, scope='EnsAdvInceptionResnetV2')
logits = (logits_v3 +1.6*logits_adv_v3+logits_v4 +logits_res_v2 + logits_resnet+logits_ens3_adv_v3+logits_ens4_adv_v3+logits_ensadv_res_v2) / 8.6
auxlogits = ( 1.6*end_points_adv_v3['AuxLogits'] +end_points_v3['AuxLogits']+end_points_ens3_adv_v3['AuxLogits']
+ end_points_v4['AuxLogits']+end_points_res_v2['AuxLogits']+end_points_ens4_adv_v3['AuxLogits']+end_points_ensadv_res_v2['AuxLogits']) / 7.6
return logits,auxlogits
def testBuildLogits(self):
batch_size = 5
height, width = 299, 299
num_classes = 1000
inputs = tf.random_uniform((batch_size, height, width, 3))
logits, end_points = inception_v4.inception_v4(inputs, num_classes)
auxlogits = end_points['AuxLogits']
predictions = end_points['Predictions']
self.assertTrue(auxlogits.op.name.startswith('InceptionV4/AuxLogits'))
self.assertListEqual(auxlogits.get_shape().as_list(),
[batch_size, num_classes])
self.assertTrue(logits.op.name.startswith('InceptionV4/Logits'))
self.assertListEqual(logits.get_shape().as_list(),
[batch_size, num_classes])
self.assertTrue(
predictions.op.name.startswith('InceptionV4/Logits/Predictions'))
self.assertListEqual(predictions.get_shape().as_list(),
[batch_size, num_classes])
