def get_vgg_loss(x, y):
from tensorflow.contrib.slim.nets import vgg as model_module
combined_images = tf.concat([x, y], axis=0)
input_img = (combined_images + 1.0) / 2.0
VGG_MEANS = np.array([[[[0.485, 0.456, 0.406]]]]).astype('float32')
VGG_MEANS = tf.constant(VGG_MEANS, shape=[1, 1, 1, 3])
vgg_input = (input_img - VGG_MEANS) * 255.0
bgr_input = tf.stack(
[vgg_input[:, :, :, 2], vgg_input[:, :, :, 1], vgg_input[:, :, :, 0]],
axis=-1)
slim = tf.contrib.slim
with slim.arg_scope(model_module.vgg_arg_scope()):
_, end_points = model_module.vgg_19(bgr_input,
num_classes=1000,
spatial_squeeze=False,
is_training=False)
loss = 0
for layer in ['vgg_19/conv3/conv3_1', 'vgg_19/conv5/conv5_1']:
layer_shape = tf.shape(end_points[layer])
x_vals = end_points[layer][:layer_shape[0] // 2]
y_vals = end_points[layer][layer_shape[0] // 2:]
loss += tf.reduce_mean(tf.pow(x_vals - y_vals, 2))
return loss
def VGG16(inputs, n_classes, freeze=False):
trainable = not freeze
inputs = tf.cast(inputs, tf.float32)
inputs = (inputs-127.5) / 127.5 # rescale: 0~255 -> -1~+1
with tf.variable_scope("vgg_16"):
with slim.arg_scope(vgg.vgg_arg_scope()):
net = slim.repeat(inputs, 2, slim.conv2d, 64, [3, 3], scope='conv1', trainable=trainable)
net = slim.max_pool2d(net, [2, 2], scope='pool1')
net = slim.repeat(net, 2, slim.conv2d, 128, [3, 3], scope='conv2', trainable=trainable)
net = slim.max_pool2d(net, [2, 2], scope='pool2')
net = slim.repeat(net, 3, slim.conv2d, 256, [3, 3], scope='conv3', trainable=trainable)
net = slim.max_pool2d(net, [2, 2], scope='pool3')
net = slim.repeat(net, 3, slim.conv2d, 512, [3, 3], scope='conv4', trainable=trainable)
net = slim.max_pool2d(net, [2, 2], scope='pool4')
net = slim.repeat(net, 3, slim.conv2d, 512, [3, 3], scope='conv5', trainable=trainable)
net = slim.max_pool2d(net, [2, 2], scope='pool5')
#Append fully connected layer
net = slim.flatten(net)
"""
net = slim.fully_connected(net, 256,
weights_initializer=tf.contrib.layers.xavier_initializer(),
weights_regularizer=slim.l2_regularizer(0.0005),
scope='finetune/fc1')
"""
net = slim.fully_connected(net, n_classes,
activation_fn=None,
weights_initializer=tf.contrib.layers.xavier_initializer(),
weights_regularizer=slim.l2_regularizer(0.001), # strong regularization
scope='finetune/classification')
return net
def Setup(self):
image_pre = vgg_preprocessing.preprocess_image(self.image,
self.image_size,
self.image_size,
is_training=False)
self.image_4d = tf.expand_dims(image_pre, 0)
# net forward
with slim.arg_scope(vgg.vgg_arg_scope()):
# 1000 classes instead of 1001.
_, _ = vgg.vgg_16(self.image_4d,
num_classes=1000,
is_training=False)
self.log("Model loading...")
self.init_fn = slim.assign_from_checkpoint_fn(
VGG16_CKPT, slim.get_model_variables('vgg_16'))
# net output
self.fc7 = tf.get_default_graph().get_tensor_by_name(
"vgg_16/fc7/Relu:0")
self.sess = tf.Session()
# variables need to be initialized before any sess.run() calls
self.init_fn(self.sess)
self.log("Restored model")
def graph(x, y, i, x_max, x_min, grad, vgg_y):
eps = FLAGS.max_epsilon
num_iter = FLAGS.num_iter
alpha = eps / num_iter
momentum = FLAGS.momentum
one_hot_vgg = tf.one_hot(vgg_y, 1000)
x_div = input_diversity(x)
with slim.arg_scope(vgg.vgg_arg_scope()):
logits, end_points = vgg.vgg_16(x_div,
num_classes=1001,
is_training=False)
cross_entropy = tf.losses.softmax_cross_entropy(one_hot_vgg, logits)
noise = tf.gradients(cross_entropy, x)[0]
noise = noise / tf.reduce_mean(tf.abs(noise), [1, 2, 3], keep_dims=True)
noise = momentum * grad + noise
x = x + alpha * tf.sign(noise)
x = tf.clip_by_value(x, x_min, x_max)
i = tf.add(i, 1)
return x, y, i, x_max, x_min, noise, vgg_y
def vgg16(inputs, is_training):
"""định nghĩa CNN
Args:
inputs: 5-D tensor [batch_size, width, height, 3]
Return:
iou: 4-D tensor [batch_size, 7, 7, 5*nbox + nclass]
"""
# khái báo scope để có thê group những biến liên quan cho việc visualize trên tensorboard.
with tf.variable_scope("vgg_16"):
with slim.arg_scope(vgg.vgg_arg_scope()):
# hàm repeat có tác dụng lặp lại tầng conv2d n lần mà không phải định nghĩa phức tạp. thank for slim package
net = slim.repeat(inputs, 2, slim.conv2d, 16, [3, 3], scope='conv1')
net = slim.max_pool2d(net, [2, 2], scope='pool1')
net = slim.repeat(net, 2, slim.conv2d, 32, [3, 3], scope='conv2')
net = slim.max_pool2d(net, [2, 2], scope='pool2')
net = slim.repeat(net, 2, slim.conv2d, 64, [3, 3], scope='conv3')
net = slim.max_pool2d(net, [2, 2], scope='pool3')
net = slim.repeat(net, 2, slim.conv2d, 128, [3, 3], scope='conv4')
net = slim.max_pool2d(net, [2, 2], scope='pool4')
net = slim.repeat(net, 2, slim.conv2d, 256, [3, 3], scope='conv5')
net = slim.max_pool2d(net, [2, 2], scope='pool5')
# thay vì sử dụng 2 tầng fully connected tại đây,
# chúng ta sử dụng conv với kernel_size = (1,1) có tác dụng giống hệt tầng fully conntected
net = slim.conv2d(net, 512, [1, 1], scope='fc6')
net = slim.conv2d(net, 13, [1, 1], activation_fn=None, scope='fc7')
return net
def _create_cnn_graph(img, is_training=True):
with tf.variable_scope("vgg_16"):
with slim.arg_scope(vgg.vgg_arg_scope()):
net = slim.repeat(img, 2, slim.conv2d, 64, [3, 3],
scope="conv1") # 创建多个拥有相同变量的指定层
net = slim.max_pool2d(net, [2, 2], scope="pool1")
net = slim.repeat(net, 2, slim.conv2d, 128, [3, 3], scope="conv2")
net = slim.max_pool2d(net, [2, 2], scope="pool2")
net = slim.repeat(net, 2, slim.conv2d, 256, [3, 3], scope="conv3")
net = slim.max_pool2d(net, [2, 2], scope="pool3")
net = slim.repeat(net, 2, slim.conv2d, 512, [3, 3], scope="conv4")
net = slim.max_pool2d(net, [2, 2], scope="pool4")
net = slim.repeat(net, 2, slim.conv2d, 512, [3, 3], scope="conv5")
net = slim.max_pool2d(net, [2, 2], scope="pool5")
net = slim.flatten(net)
net = slim.fully_connected(net,
512,
biases_regularizer=slim.l2_regularizer(0.0005),
scope="fc1")
net = slim.fully_connected(net,
2,
activation_fn=None,
biases_regularizer=slim.l2_regularizer(0.0005),
scope="fc2")
return net
def eval(x, num_classes=110):
with slim.arg_scope(inception.inception_v1_arg_scope()):
logits_inc_v1, end_points_inc_v1 = inception.inception_v1(
x, num_classes=num_classes, is_training=False, scope='InceptionV1')
pred1 = tf.argmax(end_points_inc_v1['Predictions'], 1)
# rescale pixle range from [-1, 1] to [0, 255] for resnet_v1 and vgg's input
image = (((x + 1.0) * 0.5) * 255.0)
processed_imgs_res_v1_50 = preprocess_for_model(image, 'resnet_v1_50')
with slim.arg_scope(resnet_v1.resnet_arg_scope()):
logits_res_v1_50, end_points_res_v1_50 = resnet_v1.resnet_v1_50(
processed_imgs_res_v1_50,
num_classes=num_classes,
is_training=False,
scope='resnet_v1_50')
end_points_res_v1_50['logits'] = tf.squeeze(
end_points_res_v1_50['resnet_v1_50/logits'], [1, 2])
end_points_res_v1_50['probs'] = tf.nn.softmax(
end_points_res_v1_50['logits'])
pred2 = tf.argmax(end_points_res_v1_50['probs'], 1)
# image = (((x + 1.0) * 0.5) * 255.0)#.astype(np.uint8)
processed_imgs_vgg_16 = preprocess_for_model(image, 'vgg_16')
with slim.arg_scope(vgg.vgg_arg_scope()):
logits_vgg_16, end_points_vgg_16 = vgg.vgg_16(processed_imgs_vgg_16,
num_classes=num_classes,
is_training=False,
scope='vgg_16')
end_points_vgg_16['logits'] = end_points_vgg_16['vgg_16/fc8']
end_points_vgg_16['probs'] = tf.nn.softmax(end_points_vgg_16['logits'])
pred3 = tf.argmax(end_points_vgg_16['probs'], 1)
return [pred1, pred2, pred3]
def infer(inputs, is_training=True):
inputs = tf.cast(inputs, tf.float32)
inputs = ((inputs / 255.0)-0.5)*2
#Use Pretrained Base Model
with tf.variable_scope("vgg_16"):
with slim.arg_scope(vgg.vgg_arg_scope()):
net = slim.repeat(inputs, 2, slim.conv2d, 64, [3, 3], scope='conv1')
net = slim.max_pool2d(net, [2, 2], scope='pool1')
net = slim.repeat(net, 2, slim.conv2d, 128, [3, 3], scope='conv2')
net = slim.max_pool2d(net, [2, 2], scope='pool2')
net = slim.repeat(net, 3, slim.conv2d, 256, [3, 3], scope='conv3')
net = slim.max_pool2d(net, [2, 2], scope='pool3')
net = slim.repeat(net, 3, slim.conv2d, 512, [3, 3], scope='conv4')
net = slim.max_pool2d(net, [2, 2], scope='pool4')
net = slim.repeat(net, 3, slim.conv2d, 512, [3, 3], scope='conv5')
net = slim.max_pool2d(net, [2, 2], scope='pool5')
#Append fully connected layer
net = slim.flatten(net)
net = slim.fully_connected(net, 512,
weights_initializer=tf.contrib.layers.xavier_initializer(),
weights_regularizer=slim.l2_regularizer(0.0005),
scope='finetune/fc1')
net = slim.fully_connected(net, 2,
activation_fn=None,
weights_initializer=tf.contrib.layers.xavier_initializer(),
weights_regularizer=slim.l2_regularizer(0.0005),
scope='finetune/fc2')
return net
def forward(self, input_tensor, is_training):
dropout_value = 0.5
input_tensor = tf.image.resize_images(input_tensor, [224, 224])
print("Is training:", is_training)
with slim.arg_scope(vgg.vgg_arg_scope()):
h, end_points = vgg.vgg_19(input_tensor, is_training=is_training)
print(list(end_points.keys()))
h = tf.pad(end_points['vgg_19/pool4'], [[0, 0], [1, 1], [1, 1], [0, 0]], "CONSTANT")
print(h)
h = L.convolution2d_transpose(h, 128, [5, 5], [2, 2], activation_fn=None)
h = tf.nn.relu(h)
h = L.dropout(h, keep_prob=dropout_value, is_training=is_training)
h = L.convolution2d_transpose(h, 64, [5, 5], [2, 2], activation_fn=None)
h = tf.nn.relu(h)
h = L.dropout(h, keep_prob=dropout_value, is_training=is_training)
h = L.convolution2d_transpose(h, 32, [5, 5], [2, 2], activation_fn=None)
h = tf.nn.relu(h)
h = L.dropout(h, keep_prob=dropout_value, is_training=is_training)
h = L.convolution2d_transpose(h, 32, [5, 5], [2, 2], activation_fn=None)
h = tf.nn.relu(h)
h = L.dropout(h, keep_prob=dropout_value, is_training=is_training)
h = L.convolution2d(h, len(self.classes) + 1, [1, 1], [1, 1], activation_fn=None)
return h
def vgg16(inputs, is_training):
"""định nghĩa CNN
Args:
inputs: 5-D tensor [batch_size, width, height, 3]
Return:
iou: 4-D tensor [batch_size, 7, 7, 5*nbox + nclass]
"""
#
with tf.variable_scope("vgg_16"):
with slim.arg_scope(vgg.vgg_arg_scope()):
#
net = slim.repeat(inputs,
2,
slim.conv2d,
16, [3, 3],
scope='conv1')
net = slim.max_pool2d(net, [2, 2], scope='pool1')
# TODO
net = slim.repeat(net, 2, slim.conv2d, 32, [3, 3], scope='conv2')
net = slim.max_pool2d(net, [2, 2], scope='pool2')
net = slim.repeat(net, 2, slim.conv2d, 64, [3, 3], scope='conv3')
net = slim.max_pool2d(net, [2, 2], scope='pool3')
net = slim.repeat(net, 2, slim.conv2d, 128, [3, 3], scope='conv4')
net = slim.max_pool2d(net, [2, 2], scope='pool4')
net = slim.repeat(net, 2, slim.conv2d, 256, [3, 3], scope='conv5')
net = slim.max_pool2d(net, [2, 2], scope='pool5')
net = slim.conv2d(net, 512, [1, 1], scope='fc6')
net = slim.conv2d(net, 13, [1, 1], activation_fn=None, scope='fc7')
return net
def non_target_graph(x, y, i, x_max, x_min, grad):
eps = 2.0 * max_epsilon / 255.0
alpha = eps / num_iter
num_classes = 110
with slim.arg_scope(inception.inception_v1_arg_scope()):
logits_inc_v1, end_points_inc_v1 = inception.inception_v1(
x, num_classes=num_classes, is_training=False, scope='InceptionV1')
# rescale pixle range from [-1, 1] to [0, 255] for resnet_v1 and vgg's input
image = (((x + 1.0) * 0.5) * 255.0)
processed_imgs_res_v1_50 = preprocess_for_model(image, 'resnet_v1_50')
with slim.arg_scope(resnet_v1.resnet_arg_scope()):
logits_res_v1_50, end_points_res_v1_50 = resnet_v1.resnet_v1_50(
processed_imgs_res_v1_50,
num_classes=num_classes,
is_training=False,
scope='resnet_v1_50')
end_points_res_v1_50['logits'] = tf.squeeze(
end_points_res_v1_50['resnet_v1_50/logits'], [1, 2])
end_points_res_v1_50['probs'] = tf.nn.softmax(
end_points_res_v1_50['logits'])
# image = (((x + 1.0) * 0.5) * 255.0)#.astype(np.uint8)
processed_imgs_vgg_16 = preprocess_for_model(image, 'vgg_16')
with slim.arg_scope(vgg.vgg_arg_scope()):
logits_vgg_16, end_points_vgg_16 = vgg.vgg_16(processed_imgs_vgg_16,
num_classes=num_classes,
is_training=False,
scope='vgg_16')
end_points_vgg_16['logits'] = end_points_vgg_16['vgg_16/fc8']
end_points_vgg_16['probs'] = tf.nn.softmax(end_points_vgg_16['logits'])
########################
# Using model predictions as ground truth to avoid label leaking
pred = tf.argmax(
end_points_inc_v1['Predictions'] + end_points_res_v1_50['probs'] +
end_points_vgg_16['probs'], 1)
first_round = tf.cast(tf.equal(i, 0), tf.int64)
y = first_round * pred + (1 - first_round) * y
one_hot = tf.one_hot(y, num_classes)
########################
logits = (end_points_inc_v1['Logits'] + end_points_res_v1_50['logits']
+ end_points_vgg_16['logits']) / 3.0
cross_entropy = tf.losses.softmax_cross_entropy(one_hot,
logits,
label_smoothing=0.0,
weights=1.0)
noise = tf.gradients(cross_entropy, x)[0]
noise = noise / tf.reduce_mean(tf.abs(noise), [1, 2, 3],
keep_dims=True)
noise = momentum * grad + noise
x = x + alpha * tf.sign(noise)
x = tf.clip_by_value(x, x_min, x_max)
i = tf.add(i, 1)
return x, y, i, x_max, x_min, noise
def arg_scope(self):
arg_scope_kwargs = self._config.get('arg_scope', {})
if self.vgg_type:
return vgg.vgg_arg_scope(**arg_scope_kwargs)
elif self.resnet_type:
# It's the same argscope for v1 or v2.
return resnet_v2.resnet_utils.resnet_arg_scope(**arg_scope_kwargs)
def target_graph(x, y, i, x_max, x_min, grad):
eps = 2.0 * max_epsilon / 255.0
alpha = eps / num_iter
num_classes = 110
#input image size[224,224,3]
images3 = tf.image.resize_bilinear(input_diversity(x), [224, 224], align_corners=False)
with slim.arg_scope(inception.inception_v1_arg_scope()):
logits_inc_v1, end_points_inc_v1 = inception.inception_v1(
images3, num_classes=num_classes, is_training=False, scope='InceptionV1')
# rescale pixle range from [-1, 1] to [0, 255] for resnet_v1 and vgg's input
image1 = (((input_diversity(x) + 1.0) * 0.5) * 255.0)
processed_imgs_res_v1_50 = preprocess_for_model(image1, 'resnet_v1_50')
with slim.arg_scope(resnet_v1.resnet_arg_scope()):
logits_res_v1_50, end_points_res_v1_50 = resnet_v1.resnet_v1_50(
processed_imgs_res_v1_50, num_classes=num_classes, is_training=False, scope='resnet_v1_50')
end_points_res_v1_50['logits'] = tf.squeeze(end_points_res_v1_50['resnet_v1_50/logits'], [1, 2])
end_points_res_v1_50['probs'] = tf.nn.softmax(end_points_res_v1_50['logits'])
# image = (((x + 1.0) * 0.5) * 255.0)#.astype(np.uint8)
image2 = (((input_diversity(x) + 1.0) * 0.5) * 255.0)
processed_imgs_vgg_16 = preprocess_for_model(image2, 'vgg_16')
with slim.arg_scope(vgg.vgg_arg_scope()):
logits_vgg_16, end_points_vgg_16 = vgg.vgg_16(
processed_imgs_vgg_16, num_classes=num_classes, is_training=False, scope='vgg_16')
end_points_vgg_16['logits'] = end_points_vgg_16['vgg_16/fc8']
end_points_vgg_16['probs'] = tf.nn.softmax(end_points_vgg_16['logits'])
one_hot = tf.one_hot(y, num_classes)
logits = (end_points_inc_v1['Logits'] + end_points_res_v1_50['logits'] + end_points_vgg_16['logits']) / 3.0
cross_entropy = tf.losses.softmax_cross_entropy(one_hot,
logits,
label_smoothing=0.0,
weights=1.0)
noise = tf.gradients(cross_entropy, x)[0]
noise = tf.nn.depthwise_conv2d(noise, stack_kernel, strides=[1, 1, 1, 1], padding='SAME')
noise = noise / tf.reshape(tf.contrib.keras.backend.std(tf.reshape(noise, [batch_size, -1]), axis=1),
[batch_size, 1, 1, 1])
noise = momentum * grad + noise
noise = noise / tf.reshape(tf.contrib.keras.backend.std(tf.reshape(noise, [batch_size, -1]), axis=1),
[batch_size, 1, 1, 1])
noise1 = tf.image.resize_images(noise, [140, 140], method=tf.image.ResizeMethod.NEAREST_NEIGHBOR)
print("noise shape:", noise.shape)
noise1 = alpha * tf.clip_by_value(tf.round(noise1), -2, 2)
noise_paded = tf.pad(noise1,[[0, 0], [42, 42], [42, 42], [0, 0]], constant_values=0.)
x = x - noise_paded
x = tf.clip_by_value(x, x_min, x_max)
print("x.shape:", x.shape)
i = tf.add(i, 1)
return x, y, i, x_max, x_min, noise
def target_graph(x, y, i, x_max, x_min, grad):
eps = 2.0 * max_epsilon / 255.0
alpha = eps / num_iter
num_classes = 110
with slim.arg_scope(inception.inception_v1_arg_scope()):
logits_inc_v1, end_points_inc_v1 = inception.inception_v1(
x, num_classes=num_classes, is_training=False, scope='InceptionV1')
# rescale pixle range from [-1, 1] to [0, 255] for resnet_v1 and vgg's input
image = (((x + 1.0) * 0.5) * 255.0)
processed_imgs_res_v1_50 = preprocess_for_model(image, 'resnet_v1_50')
with slim.arg_scope(resnet_v1.resnet_arg_scope()):
logits_res_v1_50, end_points_res_v1_50 = resnet_v1.resnet_v1_50(
processed_imgs_res_v1_50,
num_classes=num_classes,
is_training=False,
scope='resnet_v1_50')
end_points_res_v1_50['logits'] = tf.squeeze(
end_points_res_v1_50['resnet_v1_50/logits'], [1, 2])
end_points_res_v1_50['probs'] = tf.nn.softmax(
end_points_res_v1_50['logits'])
# image = (((x + 1.0) * 0.5) * 255.0)#.astype(np.uint8)
processed_imgs_vgg_16 = preprocess_for_model(image, 'vgg_16')
with slim.arg_scope(vgg.vgg_arg_scope()):
logits_vgg_16, end_points_vgg_16 = vgg.vgg_16(processed_imgs_vgg_16,
num_classes=num_classes,
is_training=False,
scope='vgg_16')
end_points_vgg_16['logits'] = end_points_vgg_16['vgg_16/fc8']
end_points_vgg_16['probs'] = tf.nn.softmax(end_points_vgg_16['logits'])
########################
one_hot = tf.one_hot(y, num_classes)
########################
logits = (end_points_inc_v1['Logits'] + end_points_res_v1_50['logits'] +
end_points_vgg_16['logits']) / 3.0
cross_entropy = tf.losses.softmax_cross_entropy(one_hot,
logits,
label_smoothing=0.0,
weights=1.0)
noise = tf.gradients(cross_entropy, x)[0]
noise = noise / tf.reshape(
tf.contrib.keras.backend.std(tf.reshape(noise, [batch_size, -1]),
axis=1), [batch_size, 1, 1, 1])
noise = momentum * grad + noise
noise = noise / tf.reshape(
tf.contrib.keras.backend.std(tf.reshape(noise, [batch_size, -1]),
axis=1), [batch_size, 1, 1, 1])
x = x - alpha * tf.clip_by_value(tf.round(noise), -2, 2)
x = tf.clip_by_value(x, x_min, x_max)
i = tf.add(i, 1)
return x, y, i, x_max, x_min, noise
def compute_vgg_error(output, reference, layer):
scaled_output = output * 255 - vgg_mean
scaled_reference = reference * 255 - vgg_mean
with slim.arg_scope(vgg.vgg_arg_scope()):
output_a, output_b = vgg_16(scaled_output)
reference_a, reference_b = vgg_16(scaled_reference)
if layer == 0:
return tf.abs(output_a - reference_a)
return tf.abs(output_b - reference_b)
def get_vgg_activations(input,
layers=[
'vgg_19/conv2/conv2_1', 'vgg_19/conv3/conv3_1',
'vgg_19/conv4/conv4_1'
]):
with slim.arg_scope(vgg.vgg_arg_scope()):
_, end_points = vgg_19(input)
activations = [end_points[layer] for layer in layers]
return activations
def build_single_vggnet(train_tfdata, is_train, dropout_keep_prob):
if not FLAGS.is_train or FLAGS.debug_test:
is_train = False
with slim.arg_scope(vgg.vgg_arg_scope()):
identity, end_points = vgg.vgg_16(train_tfdata, num_classes=FLAGS.num_class, is_training=is_train, dropout_keep_prob = dropout_keep_prob)
# identity, end_points = vgg.vgg_19(train_tfdata, num_classes=FLAGS.num_class, is_training=is_train, dropout_keep_prob = dropout_keep_prob)
for key in end_points.keys():
if 'fc7' in key:
feature = tf.squeeze(end_points[key], [1, 2])
return identity, feature
def netvgg(inputs, is_training = True):
inputs = tf.cast(inputs, tf.float32)
inputs = ((inputs / 255.0) -0.5)*2
num_anchors = 9
with tf.variable_scope("vgg_16"):
with slim.arg_scope(vgg.vgg_arg_scope()):
net = inputs
net = slim.repeat(net, 2, slim.conv2d, 64, [3, 3], scope = 'conv1')
net = slim.max_pool2d(net, [2, 2], scope = 'pool1')
net = slim.repeat(net, 2, slim.conv2d, 128, [3, 3], scope='conv2')
net = slim.max_pool2d(net, [2, 2], scope = 'pool2')
net = slim.repeat(net, 3, slim.conv2d, 256, [3, 3], scope='conv3')
net = slim.max_pool2d(net, [2, 2], scope = 'pool3')
net = slim.repeat(net, 3, slim.conv2d, 512, [3, 3], scope='conv4')
net = slim.max_pool2d(net, [2, 2], scope = 'pool4')
net = slim.repeat(net, 3, slim.conv2d, 512, [3, 3], scope='conv5')
initializer = tf.random_normal_initializer(mean=0.0, stddev=0.01)
net_cnn = net
net1 = slim.conv2d(net, 2*num_anchors, [1, 1], scope= "prob",
weights_initializer=initializer, activation_fn=None)#tf.nn.sigmoid)
net1 = Reshape((-1, 2), input_shape=(net1.shape[1], net1.shape[2], net1.shape[3]))(net1)
#net1 = tf.reshape(net1, [0, -1, -1, 2])
input_shape = tf.shape(net1)
rshpsssss = tf.reshape(net1, [-1, input_shape[-1]])
print "rshpsssss", rshpsssss
#net1 = tf.nn.softmax(net1)
net2 = slim.conv2d(net, 4*num_anchors, [1, 1], scope='bbox', weights_initializer=initializer, activation_fn=None)
net2 = Reshape((-1, 4), input_shape=(net2.shape[1], net2.shape[2], net2.shape[3]))(net2)
net = slim.max_pool2d(net, [2, 2], scope = 'pool5')
net = slim.flatten(net)
w_init = tf.contrib.layers.xavier_initializer()
w_reg = slim.l2_regularizer(0.0005)
net = slim.fully_connected(net, 4096,
weights_initializer = w_init,
weights_regularizer = w_reg,
scope = 'fc6')
net = slim.dropout(net, keep_prob = 0.5, is_training = is_training)
net = slim.fully_connected(net, 4096,
weights_initializer = w_init,
weights_regularizer = w_reg,
scope = 'fc7')
net = slim.dropout(net, keep_prob = 0.5, is_training = is_training)
net = slim.fully_connected(net, 1000,
weights_initializer = w_init,
weights_regularizer = w_reg,
scope = 'fc8')
print("SHAPE!!!!", net)
return net_cnn, net2, net1, net
def arg_scope(self):
arg_scope_kwargs = self._config.get('arg_scope', {})
if self.vgg_type:
return vgg.vgg_arg_scope(**arg_scope_kwargs)
if self.resnet_type:
# It's the same arg_scope for v1 or v2.
return resnet_v2.resnet_utils.resnet_arg_scope(**arg_scope_kwargs)
raise ValueError('Invalid architecture: "{}"'.format(
self._config.get('architecture')))
def __init__(self, inputs, true_labels, is_train=False, num_classes=None):
self.true_labels = true_labels
self.NUM_CLASSES = num_classes
with slim.arg_scope(vgg.vgg_arg_scope()):
self.output, self.features = vgg.vgg_16(inputs=inputs,
num_classes=1000,
is_training=False)
self.classifier, _ = cnn.fc(input=self.extract_features('fc7'),
num_outputs=self.NUM_CLASSES,
use_relu=False,
name='classifier')
def forward(self, input_tensor, is_training):
dropout_value = 0.5
# input_tensor = tf.image.resize_images(input_tensor, [224, 224])
batch_size = tf.shape(input_tensor)[0]
print("Is training:", is_training)
with slim.arg_scope(vgg.vgg_arg_scope()):
h, end_points = vgg.vgg_16(input_tensor, is_training=is_training)
print(end_points)
print(list(end_points.keys()))
h = end_points['vgg_16/pool4']
h = L.convolution2d_transpose(h, 256, [5, 5], [2, 2], activation_fn=None)
h = tf.nn.relu(h)
h = tf.concat([h, end_points['vgg_16/pool3']], axis=3)
h = L.dropout(h, keep_prob=dropout_value, is_training=is_training)
np_seed_mask = np.zeros((1, 56, 56, 1))
np_seed_mask[:, 28:29, 28:29, :] = 1.0
seed_mask = tf.constant(np_seed_mask, dtype=tf.float32)
seed_mask = tf.tile(seed_mask, [batch_size, 1, 1, 1])
h = L.convolution2d_transpose(h, 128, [5, 5], [2, 2], activation_fn=None)
h = tf.nn.relu(h)
h = tf.concat([h, end_points['vgg_16/pool2'], seed_mask], axis=3)
h = L.dropout(h, keep_prob=dropout_value, is_training=is_training)
h = L.convolution2d_transpose(h, 64, [5, 5], [2, 2], activation_fn=None)
h = tf.nn.relu(h)
h = tf.concat([h, end_points['vgg_16/pool1']], axis=3)
h = L.dropout(h, keep_prob=dropout_value, is_training=is_training)
h = L.convolution2d_transpose(h, 64, [5, 5], [2, 2], activation_fn=None)
h = tf.concat([h, input_tensor], axis=3)
h = tf.nn.relu(h)
h = L.dropout(h, keep_prob=dropout_value, is_training=is_training)
# h = L.convolution2d_transpose(h, 64, [5, 5], [2, 2], activation_fn=None)
# h = tf.nn.relu(h)
# h = L.dropout(h, keep_prob=dropout_value, is_training=is_training)
# h = L.convolution2d_transpose(h, 64, [5, 5], [2, 2], activation_fn=None)
# h = tf.nn.relu(h)
# h = L.dropout(h, keep_prob=dropout_value, is_training=is_training)
h = L.convolution2d(h, len(self.classes) + 1, [1, 1], [1, 1], activation_fn=None)
return h
def build_single_vggnet(train_tfdata, is_train, dropout_keep_prob):
if not FLAGS.is_train or FLAGS.debug_test:
is_train = False
with slim.arg_scope(vgg.vgg_arg_scope()):
identity, end_points = vgg.vgg_16(train_tfdata,
num_classes=FLAGS.num_class,
is_training=is_train,
dropout_keep_prob=dropout_keep_prob)
# identity, end_points = vgg.vgg_19(train_tfdata, num_classes=FLAGS.num_class, is_training=is_train, dropout_keep_prob = dropout_keep_prob)
for key in end_points.keys():
if 'fc7' in key:
feature = tf.squeeze(end_points[key], [1, 2])
return identity, feature
def fprop(self, x, **kwargs):
del kwargs
with tf.variable_scope(self.scope, reuse=tf.AUTO_REUSE):
with slim.arg_scope(vgg.vgg_arg_scope()):
logits, _ = vgg.vgg_19(
x, num_classes=self.nb_classes,
dropout_keep_prob=self.dropout_keep_prob,
is_training=self.is_training, scope=self.scope)
probs = tf.nn.softmax(logits)
return {self.O_LOGITS: logits, self.O_PROBS: probs}
def arg_scope(self):
arg_scope_kwargs = self._config.get('arg_scope', {})
if self.vgg_type:
return vgg.vgg_arg_scope(**arg_scope_kwargs)
if self.resnet_type:
# It's the same arg_scope for v1 or v2.
return resnet_v2.resnet_utils.resnet_arg_scope(**arg_scope_kwargs)
raise ValueError('Invalid architecture: "{}"'.format(
self._config.get('architecture')
))
def netvgg(inputs, is_training=True):
"""
Output: probability and regression
net1: probability
net2: regression
"""
inputs = tf.cast(inputs, tf.float32)
inputs = ((inputs / 255.0) - 0.5) * 2
num_anchors = 9
with tf.variable_scope("vgg_16"):
with slim.arg_scope(vgg.vgg_arg_scope()):
net = inputs
net = slim.repeat(net, 2, slim.conv2d, 64, [3, 3], scope='conv1')
net = slim.max_pool2d(net, [2, 2], scope='pool1')
net = slim.repeat(net, 2, slim.conv2d, 128, [3, 3], scope='conv2')
net = slim.max_pool2d(net, [2, 2], scope='pool2')
net = slim.repeat(net, 3, slim.conv2d, 256, [3, 3], scope='conv3')
net = slim.max_pool2d(net, [2, 2], scope='pool3')
net = slim.repeat(net, 3, slim.conv2d, 512, [3, 3], scope='conv4')
net = slim.max_pool2d(net, [2, 2], scope='pool4')
net = slim.repeat(net, 3, slim.conv2d, 512, [3, 3], scope='conv5')
initializer = tf.random_normal_initializer(mean=0.0, stddev=0.01)
net_cnn = net
net1 = slim.conv2d(net,
2 * num_anchors, [1, 1],
scope="prob",
weights_initializer=initializer,
activation_fn=None) #tf.nn.sigmoid)
net1 = Reshape((-1, 2),
input_shape=(net1.shape[1], net1.shape[2],
net1.shape[3]))(net1)
#net1 = tf.reshape(net1, [0, -1, -1, 2])
input_shape = tf.shape(net1)
#rshpsssss = tf.reshape(net1, [-1, input_shape[-1]])
#print "rshpsssss", rshpsssss
#net1 = tf.nn.softmax(net1)
net2 = slim.conv2d(net,
4 * num_anchors, [1, 1],
scope='bbox',
weights_initializer=initializer,
activation_fn=None)
net2 = Reshape((-1, 4),
input_shape=(net2.shape[1], net2.shape[2],
net2.shape[3]))(net2)
net = slim.max_pool2d(net, [2, 2], scope='pool5')
return net_cnn, net2, net1
def __call__(self, x_input, return_logits=False):
"""Constructs model and return probabilities for given input."""
reuse = True if self.built else None
with slim.arg_scope(vgg.vgg_arg_scope()):
_, end_points = vgg.vgg_16(
x_input, num_classes=self.nb_classes, is_training=False)
self.built = True
self.logits = tf.squeeze(end_points['vgg_16/fc8'])
# Strip off the extra reshape op at the output
self.probs = tf.nn.softmax(end_points['vgg_16/fc8'])
if return_logits:
return self.logits
else:
return self.probs
def __call__(self, ens_x_input, vgg_x_input, inc_x_input, tcd_x_input):
"""Constructs model and return probabilities for given input."""
reuse = True if self.built else None
logits = None
aux_logits = None
weights = [[0.7, 0.1], [0.2, 0.1]]
all_inputs = [[ens_x_input, tcd_x_input], [inc_x_input, tcd_x_input]]
scopes = [
inception_resnet_v2.inception_resnet_v2_arg_scope(),
inception.inception_v3_arg_scope()
]
reuse_flags = [reuse, True]
for model_idx, model in enumerate(
[inception_resnet_v2.inception_resnet_v2, inception.inception_v3]):
with slim.arg_scope(scopes[model_idx]):
for idx, inputs in enumerate(all_inputs[model_idx]):
result = model(inputs,
num_classes=self.num_classes,
is_training=False,
reuse=reuse_flags[idx])
weight = weights[model_idx][idx]
# :1 is for slicing out the background class
if logits == None:
logits = result[0][:, 1:] * weight
aux_logits = result[1]['AuxLogits'][:, 1:] * weight
else:
logits += result[0][:, 1:] * weight
aux_logits += result[1]['AuxLogits'][:, 1:] * weight
with slim.arg_scope(vgg.vgg_arg_scope()):
weight = 0.1
result = vgg.vgg_16(vgg_x_input,
num_classes=1000,
is_training=False)
logits += result[0] * weight
with slim.arg_scope(resnet_utils.resnet_arg_scope()):
weight = 0.05
result = resnet_v2.resnet_v2_152(vgg_x_input,
num_classes=self.num_classes,
reuse=reuse)
logits += tf.squeeze(result[0])[:, 1:] * weight
self.built = True
aux_weight = 0.8
logits += aux_logits * aux_weight
predictions = layers_lib.softmax(logits)
return predictions
def main():
with tf.gfile.FastGFile('output_graph.pb', 'rb') as f:
graph_def = tf.GraphDef()
graph_def.ParseFromString(f.read())
tf.import_graph_def(graph_def, name='')
with tf.Graph().as_default() as graph:
inputs = tf.placeholder(dtype=tf.float32, shape=[1, 224, 224, 3])
with slim.arg_scope(vgg.vgg_arg_scope()):
_, end_points = vgg.vgg_16(inputs,
num_classes=1000,
is_training=True,
dropout_keep_prob=0.5,
spatial_squeeze=False,
scope='vgg_16')
stats_graph(graph)
def Deeplab_v1(inputs, num_classes):
'''A TensorFlow implementation of Deeplab_v1 model based on
http://liangchiehchen.com/projects/DeepLab-LargeFOV.html
Args:
inputs: A 4-D tensor with dimensions [batch_size, height, width, channels]
num_classes: Integer, the total number of categories in the dataset
Returns:
A score map with dimensions [batch_size, 1/8*height, 1/8*width, num_classes]
'''
with slim.arg_scope(vgg.vgg_arg_scope()):
_, end_points = vgg.vgg_16(inputs,
num_classes=1000,
is_training=True,
dropout_keep_prob=0.5,
spatial_squeeze=False,
scope='vgg_16')
conv4 = end_points["vgg_16/conv4/conv4_3"] # 1/8H * 1/8 * 256
with tf.variable_scope('Deeplab_v1'):
pool4 = slim.max_pool2d(conv4, [3, 3],
stride=1,
padding='SAME',
scope='pool4')
conv5_1 = slim.conv2d(pool4, 512, [3, 3], rate=2, scope='conv5_1')
conv5_2 = slim.conv2d(conv5_1, 512, [3, 3], rate=2, scope='conv5_2')
conv5_3 = slim.conv2d(conv5_2, 512, [3, 3], rate=2, scope='conv5_3')
pool5_1 = slim.max_pool2d(conv5_3, [3, 3],
stride=1,
padding='SAME',
scope='pool5_1')
pool5_2 = slim.avg_pool2d(pool5_1, [3, 3],
stride=1,
padding='SAME',
scope='pool5_2')
conv6 = slim.conv2d(pool5_2, 1024, [3, 3], rate=12, scope='conv6')
dropout1 = slim.dropout(conv6, keep_prob=0.5)
conv7 = slim.conv2d(dropout1, 1024, [1, 1], scope='conv7')
dropout2 = slim.dropout(conv7, keep_prob=0.5)
logits = slim.conv2d(dropout2,
num_classes, [1, 1],
activation_fn=None,
scope='logits')
return logits
def get_predicted_y(x, num_classes, args):
"""Calculate predicted label"""
slim = tf.contrib.slim
with slim.arg_scope(inception.inception_v1_arg_scope()):
logits_inc_v1, end_points_inc_v1 = inception.inception_v1(
dimension_224(x),
num_classes=num_classes,
is_training=False,
scope='InceptionV1')
image = (((x + 1.0) * 0.5) * 255.0)
processed_imgs_res_v1_50 = preprocess_for_model(image, 'resnet_v1_50')
with slim.arg_scope(resnet_v1.resnet_arg_scope()):
logits_res_v1_50, end_points_res_v1_50 = resnet_v1.resnet_v1_50(
dimension_224(processed_imgs_res_v1_50),
num_classes=num_classes,
is_training=False,
scope='resnet_v1_50')
end_points_res_v1_50['logits'] = tf.squeeze(
end_points_res_v1_50['resnet_v1_50/logits'], [1, 2])
end_points_res_v1_50['probs'] = tf.nn.softmax(
end_points_res_v1_50['logits'])
processed_imgs_vgg_16 = preprocess_for_model(image, 'vgg_16')
with slim.arg_scope(vgg.vgg_arg_scope()):
logits_vgg_16, end_points_vgg_16 = vgg.vgg_16(
dimension_224(processed_imgs_vgg_16),
num_classes=num_classes,
is_training=False,
scope='vgg_16')
end_points_vgg_16['logits'] = end_points_vgg_16['vgg_16/fc8']
end_points_vgg_16['probs'] = tf.nn.softmax(end_points_vgg_16['logits'])
logits = [None] * args.num_model
pred_y = [None] * args.num_model
logits[0] = end_points_inc_v1['Logits']
logits[1] = end_points_res_v1_50['logits']
logits[2] = end_points_vgg_16['logits']
for i in range(args.num_model):
pred_y[i] = tf.argmax(tf.nn.softmax(logits[i]), 1)
return pred_y, logits
def ens_model(x):
#input remains in[0,1]
image = (x * 255.0)
num_classes = 110
processed_incv1 = preprocess_for_model(image, 'inception_v1')
with slim.arg_scope(inception.inception_v1_arg_scope()):
logits_inc_v1, end_points_inc_v1 = inception.inception_v1(
processed_incv1,
num_classes=num_classes,
is_training=False,
scope='InceptionV1')
processed_imgs_res_v1_50 = preprocess_for_model(image, 'resnet_v1_50')
with slim.arg_scope(resnet_v1.resnet_arg_scope()):
logits_res_v1_50, end_points_res_v1_50 = resnet_v1.resnet_v1_50(
processed_imgs_res_v1_50,
num_classes=num_classes,
is_training=False,
scope='resnet_v1_50')
end_points_res_v1_50['logits'] = tf.squeeze(
end_points_res_v1_50['resnet_v1_50/logits'], [1, 2])
end_points_res_v1_50['probs'] = tf.nn.softmax(
end_points_res_v1_50['logits'])
# image = (((x + 1.0) * 0.5) * 255.0)#.astype(np.uint8)
processed_imgs_vgg_16 = preprocess_for_model(image, 'vgg_16')
with slim.arg_scope(vgg.vgg_arg_scope()):
logits_vgg_16, end_points_vgg_16 = vgg.vgg_16(processed_imgs_vgg_16,
num_classes=num_classes,
is_training=False,
scope='vgg_16')
end_points_vgg_16['logits'] = end_points_vgg_16['vgg_16/fc8']
end_points_vgg_16['probs'] = tf.nn.softmax(end_points_vgg_16['logits'])
########################
#one_hot = tf.one_hot(y, num_classes)
########################
logits = (end_points_inc_v1['Logits'] + end_points_res_v1_50['logits'] +
end_points_vgg_16['logits']) / 3.0
print('logits.shape:', logits.shape)
return logits
def test_vgg(self):
with slim.arg_scope(vgg.vgg_arg_scope()):
net, end_points = vgg.vgg_16(self.inputs,
self.nbclasses,
is_training=False)
net = slim.softmax(net)
saver = tf.train.Saver(tf.global_variables())
check_point = 'test/data/vgg_16.ckpt'
sess = tf.InteractiveSession()
saver.restore(sess, check_point)
self.sess = sess
self.graph_origin = tf.get_default_graph()
self.target_op_name = darkon.Gradcam.candidate_featuremap_op_names(
sess, self.graph_origin)[-2]
self.model_name = 'vgg'