def create_model(self,
images,
num_classes,
weight_decay=0.00004,
scope='Flowers',
reuse=None,
is_training=True):
"""Creates a base part of the Model (no gradients, no loss, no summaries).
Args:
images: A tensor of size [batch_size, height, width, channels].
num_classes: The number of predicted classes.
scope: Optional variable_scope.
reuse: Whether or not the network or its variables should be reused. To
be able to reuse 'scope' must be given.
is_training: Whether is training or not.
Returns:
A named tuple OutputEndpoints.
"""
with tf.variable_scope(scope, [images], reuse=reuse):
with slim.arg_scope(inception_v3.inception_v3_arg_scope(weight_decay=weight_decay)):
logits, endpoints = inception_v3.inception_v3(
inputs = images,
num_classes=num_classes,
is_training=is_training)
return logits, endpoints
def network_fn(inputs):
# return transformer_factory.transform(inputs, BATCH_PER_GPU, NUM_STN, (224, 224), NUM_CLASSES, FLAGS.weight_decay, True)
end_points = {}
# with slim.arg_scope([slim.batch_norm, slim.dropout], is_training=True):
# with slim.arg_scope(inception_v3_arg_scope(weight_decay=FLAGS.weight_decay)):
with slim.arg_scope([slim.batch_norm, slim.dropout],
is_training=is_training):
with slim.arg_scope(inception_v3_arg_scope(weight_decay=weight_decay)):
with tf.variable_scope("loc") as scope:
with tf.variable_scope("net") as scope2:
# _, _end_points = inception_resnet_v2.inception_resnet_v2(inputs, num_classes=2, is_training=True, scope = scope2)
loc_net, _ = inception_v2.inception_v2_base(inputs,
scope=scope2)
# loc_net = _end_points['Conv2d_7b_1x1']
loc_net = slim.conv2d(loc_net, 128, [1, 1], scope='Loc_1x1')
default_kernel_size = [14, 14]
# kernel_size = _reduced_kernel_size_for_small_input(loc_net, default_kernel_size)
loc_net = slim.conv2d(loc_net,
128,
loc_net.get_shape()[1:3],
padding='VALID',
activation_fn=tf.nn.tanh,
scope='Loc_fc1')
loc_net = slim.flatten(loc_net)
iv = 4.
initial = np.array([iv, 0, iv, 0] * NUM_STN, dtype=np.float32)
b_fc_loc = tf.get_variable(
"Loc_fc_b",
shape=[4 * NUM_STN],
initializer=init_ops.constant_initializer(initial),
dtype=dtypes.float32)
W_fc_loc = tf.get_variable(
"Loc_fc_W",
shape=[128, 4 * NUM_STN],
initializer=init_ops.constant_initializer(
np.zeros((128, 4 * NUM_STN))),
dtype=dtypes.float32)
theta = tf.nn.tanh(tf.matmul(loc_net, W_fc_loc) + b_fc_loc)
_finals = []
for i in xrange(NUM_STN):
scope_name = "stn%d" % i
with tf.variable_scope(scope_name) as scope1:
_theta = tf.slice(theta, [0, 4 * i], [-1, 4 * (i + 1)])
# loc_net = slim.conv2d(loc_net, 6, [1,1], activation_fn=tf.nn.tanh, scope='Loc_fc', biases_initializer = init_ops.constant_initializer([4.0,0.0,0.0,0.0,4.0,0.0]*128,dtype=dtypes.float32))
# loc_net = slim.conv2d(loc_net, 6, [1,1], activation_fn=tf.nn.tanh, scope='Loc_fc', biases_initializer = init_ops.constant_initializer([4.0],dtype=dtypes.float32))
# loc_net = slim.flatten(loc_net)
stn_output_size = (STN_OUT_SIZE, STN_OUT_SIZE)
x = transformer(inputs, _theta, stn_output_size)
x.set_shape([
BATCH_PER_GPU, stn_output_size[0], stn_output_size[1],
3
])
# x.set_shape(tf.shape(inputs))
# tf.reshape(x, tf.shape(inputs))
end_points['x'] = x
# with tf.variable_scope("net") as scope2:
# return inception_resnet_v2.inception_resnet_v2(x, num_classes=NUM_CLASSES, is_training=True, scope = scope2)
with tf.variable_scope("net") as scope2:
net, _ = inception_v2.inception_v2_base(x,
scope=scope2)
kernel_size = _reduced_kernel_size_for_small_input(
net, [7, 7])
net = slim.avg_pool2d(net,
kernel_size,
padding='VALID',
scope='AvgPool_1a')
net = slim.dropout(net, keep_prob=0.7, scope='Dropout_1b')
_finals.append(net)
with tf.variable_scope('Logits'):
net = tf.concat(axis=3, values=_finals)
logits = slim.conv2d(net,
NUM_CLASSES, [1, 1],
activation_fn=None,
normalizer_fn=None,
scope='Conv2d_1c_1x1')
logits = tf.squeeze(logits, [1, 2], name='SpatialSqueeze')
predictions = slim.softmax(logits, scope='Predictions')
end_points['Predictions'] = predictions
logits_a = slim.conv2d(net,
NUM_ATTRIBS, [1, 1],
activation_fn=None,
normalizer_fn=None,
scope='Conv2d_1c_1x1_a')
logits_a = tf.squeeze(logits_a, [1, 2],
name='SpatialSqueeze_a')
predictions_a = slim.sigmoid(logits_a, scope='Predictions_a')
end_points['Predictions_a'] = predictions_a
return logits, logits_a, end_points
def graph(x, y, i, x_max, x_min, accum_s, accum_g):
eps = 2.0 * FLAGS.max_epsilon / 255.0
num_iter = FLAGS.num_iter
alpha = eps / num_iter
# momentum = FLAGS.momentum
num_classes = 1001
beta_1 = FLAGS.beta_1
beta_2 = FLAGS.beta_2
x_nes = x
with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
logits_v3, end_points_v3 = inception_v3.inception_v3(
x_nes,
num_classes=num_classes,
is_training=False,
reuse=tf.AUTO_REUSE)
pred = tf.argmax(end_points_v3['Predictions'], 1)
first_round = tf.cast(tf.equal(i, 0), tf.int64)
y = first_round * pred + (1 - first_round) * y
one_hot = tf.one_hot(y, num_classes)
cross_entropy = tf.losses.softmax_cross_entropy(one_hot, logits_v3)
grad = tf.gradients(cross_entropy, x)[0]
x_nes_2 = 1 / 2 * x_nes
with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
logits_v3_2, end_points_v3 = inception_v3.inception_v3(
x_nes_2,
num_classes=num_classes,
is_training=False,
reuse=tf.AUTO_REUSE)
cross_entropy_2 = tf.losses.softmax_cross_entropy(one_hot, logits_v3_2)
grad += tf.gradients(cross_entropy_2, x)[0]
x_nes_4 = 1 / 4 * x_nes
with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
logits_v3_4, end_points_v3 = inception_v3.inception_v3(
x_nes_4,
num_classes=num_classes,
is_training=False,
reuse=tf.AUTO_REUSE)
cross_entropy_4 = tf.losses.softmax_cross_entropy(one_hot, logits_v3_4)
grad += tf.gradients(cross_entropy_4, x)[0]
x_nes_8 = 1 / 8 * x_nes
with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
logits_v3_8, end_points_v3 = inception_v3.inception_v3(
x_nes_8,
num_classes=num_classes,
is_training=False,
reuse=tf.AUTO_REUSE)
cross_entropy_8 = tf.losses.softmax_cross_entropy(one_hot, logits_v3_8)
grad += tf.gradients(cross_entropy_8, x)[0]
x_nes_16 = 1 / 16 * x_nes
with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
logits_v3_16, end_points_v3 = inception_v3.inception_v3(
x_nes_16,
num_classes=num_classes,
is_training=False,
reuse=tf.AUTO_REUSE)
cross_entropy_16 = tf.losses.softmax_cross_entropy(one_hot, logits_v3_16)
grad += tf.gradients(cross_entropy_16, x)[0]
grad_normed = grad / tf.reduce_mean(tf.abs(grad), [1, 2, 3],
keep_dims=True)
accum_g = grad_normed * (1 - beta_1) + accum_g * beta_1
accum_s = tf.multiply(grad, grad) * (1 - beta_2) + accum_s * beta_2
accum_g_hat = tf.divide(accum_g,
(1 - tf.pow(beta_1, tf.cast(i + 1, tf.float32))))
accum_s_hat = tf.divide(accum_s,
(1 - tf.pow(beta_2, tf.cast(i + 1, tf.float32))))
x = x + tf.multiply(tf.divide(alpha, tf.add(tf.sqrt(accum_s_hat), 1e-6)),
tf.sign(accum_g_hat))
x = tf.clip_by_value(x, x_min, x_max)
i = tf.add(i, 1)
return x, y, i, x_max, x_min, accum_s, accum_g
'''
def getimagedir(path):
res = []
for root, dirs, files in os.walk(path):
rootpath = os.path.join(os.path.abspath(path), root)
for file in files:
filepath = os.path.join(rootpath, file)
res.append(filepath)
return res
'''
with tf.Graph().as_default():
with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
# set your image
#imgPath = '/mnt/notebook/kihong/models/slim/images/dog.jpg'
imgPath = 'images/dog2.jpg'
testImage_string = tf.gfile.FastGFile(imgPath, 'rb').read()
testImage = tf.image.decode_jpeg(testImage_string, channels=3)
processed_image = inception_preprocessing.preprocess_image(testImage, image_size, image_size, is_training=False)
processed_images = tf.expand_dims(processed_image, 0)
logits, _ = inception_v3.inception_v3(processed_images, num_classes=257, is_training=False)
probabilities = tf.nn.softmax(logits)
init_fn = slim.assign_from_checkpoint_fn(
os.path.join(checkpoints_dir, 'model.ckpt-500'), slim.get_model_variables('InceptionV3'))
with tf.Session() as sess:
def graph(x, y, i, x_max, x_min, g1, g2, e):
eps = 2.0 * FLAGS.max_epsilon / 255.0
num_iter = FLAGS.num_iter
alpha = eps / num_iter
num_classes = 1001
beta = 0.91
k1 = 0.99
k2 = 0.72
k3 = 1.44
with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
logits_v3, end_points_v3 = inception_v3.inception_v3(
input_diversity(x),
num_classes=num_classes,
is_training=False,
reuse=tf.AUTO_REUSE)
with slim.arg_scope(inception_v4.inception_v4_arg_scope()):
logits_v4, end_points_v4 = inception_v4.inception_v4(
input_diversity(x),
num_classes=num_classes,
is_training=False,
reuse=tf.AUTO_REUSE)
with slim.arg_scope(inception_resnet_v2.inception_resnet_v2_arg_scope()):
logits_res_v2, end_points_res_v2 = inception_resnet_v2.inception_resnet_v2(
input_diversity(x),
num_classes=num_classes,
is_training=False,
reuse=tf.AUTO_REUSE)
with slim.arg_scope(resnet_v2.resnet_arg_scope()):
logits_resnet, end_points_resnet = resnet_v2.resnet_v2_101(
input_diversity(x),
num_classes=num_classes,
is_training=False,
scope='resnet_v2_101',
reuse=tf.AUTO_REUSE)
logits = (logits_v3 + logits_v4 + logits_res_v2 + logits_resnet) / 4
auxlogits = (end_points_v3['AuxLogits'] + end_points_v4['AuxLogits'] +
end_points_res_v2['AuxLogits']) / 3
cross_entropy = tf.losses.softmax_cross_entropy(y,
logits,
label_smoothing=0.0,
weights=1.0)
cross_entropy += tf.losses.softmax_cross_entropy(y,
auxlogits,
label_smoothing=0.0,
weights=0.4)
noise = tf.gradients(cross_entropy, x)[0]
noise = tf.nn.depthwise_conv2d(noise,
stack_kernel,
strides=[1, 1, 1, 1],
padding='SAME')
noise = noise / tf.reduce_mean(tf.abs(noise), [1, 2, 3], keep_dims=True)
g1 = beta * g1 + (1 - beta) * (noise - e)
g2 = g2 + noise
x = x + alpha * tf.sign(k1 * noise + k2 * g1 + k3 * g2)
x = tf.clip_by_value(x, x_min, x_max)
i = tf.add(i, 1)
e = noise
return x, y, i, x_max, x_min, g1, g2, e
def graph(x, y, i, x_max, x_min, grad, y_target, y_logits):
eps = 2.0 * FLAGS.max_epsilon / 255.0
num_iter = FLAGS.num_iter
alpha = eps / num_iter
momentum = FLAGS.momentum
num_classes = 1001
# should keep original x here for output
with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
logits_v3, end_points_v3 = inception_v3.inception_v3(
input_diversity(x), num_classes=num_classes, is_training=False)
with slim.arg_scope(inception_v4.inception_v4_arg_scope()):
logits_v4, end_points_v4 = inception_v4.inception_v4(
input_diversity(x), num_classes=num_classes, is_training=False)
with slim.arg_scope(inception_resnet_v2.inception_resnet_v2_arg_scope()):
logits_res_v2, end_points_res_v2 = inception_resnet_v2.inception_resnet_v2(
input_diversity(x), num_classes=num_classes, is_training=False)
with slim.arg_scope(resnet_v2.resnet_arg_scope()):
logits_resnet, end_points_resnet = resnet_v2.resnet_v2_152(
input_diversity(x),
num_classes=num_classes,
is_training=False,
reuse=tf.AUTO_REUSE)
with slim.arg_scope(resnet_v2.resnet_arg_scope()):
logits_resnet_50, end_points_resnet_50 = resnet_v2.resnet_v2_50(
input_diversity(x),
num_classes=num_classes,
is_training=False,
reuse=tf.AUTO_REUSE)
with slim.arg_scope(resnet_v2.resnet_arg_scope()):
logits_resnet_101, end_points_resnet_101 = resnet_v2.resnet_v2_101(
input_diversity(x),
num_classes=num_classes,
is_training=False,
reuse=tf.AUTO_REUSE)
# Adv training models
# with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
# logits_adv_v3, end_points_adv_v3 = inception_v3.inception_v3(
# input_diversity(x), num_classes=num_classes, is_training=False, scope='AdvInceptionV3')
#
# with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
# logits_ens3_adv_v3, end_points_ens3_adv_v3 = inception_v3.inception_v3(
# input_diversity(x), num_classes=num_classes, is_training=False, scope='Ens3AdvInceptionV3')
with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
logits_ens4_adv_v3, end_points_ens4_adv_v3 = inception_v3.inception_v3(
input_diversity(x),
num_classes=num_classes,
is_training=False,
scope='Ens4AdvInceptionV3')
with slim.arg_scope(inception_resnet_v2.inception_resnet_v2_arg_scope()):
logits_ensadv_res_v2, end_points_ensadv_res_v2 = inception_resnet_v2.inception_resnet_v2(
input_diversity(x),
num_classes=num_classes,
is_training=False,
scope='EnsAdvInceptionResnetV2')
logits = (logits_v4 + logits_res_v2 + logits_v3 + logits_resnet +
logits_resnet_50 + logits_resnet_101 + logits_ens4_adv_v3 +
logits_ensadv_res_v2) / 8
auxlogits = (end_points_v4['AuxLogits'] + end_points_res_v2['AuxLogits'] +
end_points_v3['AuxLogits'] +
end_points_ens4_adv_v3['AuxLogits'] +
end_points_ensadv_res_v2['AuxLogits']) / 5
y_oh = tf.one_hot(y, num_classes)
y_target_oh = tf.one_hot(y_target, num_classes)
loss = -FLAGS.W_crs * tf.losses.softmax_cross_entropy(
y_oh, logits, label_smoothing=0.0, weights=1.0)
# loss = - Poincare_dis(tf.clip_by_value((y_oh-0.01), 0.0, 1.0),
# logits / tf.reduce_sum(tf.abs(logits), [1], keep_dims=True) )
loss_ce = tf.losses.softmax_cross_entropy(y_target_oh,
logits,
label_smoothing=0.0,
weights=1.0)
loss_ce += tf.losses.softmax_cross_entropy(y_target_oh,
auxlogits,
label_smoothing=0.0,
weights=0.9)
loss_po = Poincare_dis(
tf.clip_by_value((y_target_oh - 0.00001), 0.0, 1.0),
logits / tf.reduce_sum(tf.abs(logits), [1], keep_dims=True))
loss_po += FLAGS.W_aux * Poincare_dis(
tf.clip_by_value((y_target_oh - 0.00001), 0.0, 1.0),
auxlogits / tf.reduce_sum(tf.abs(auxlogits), [1], keep_dims=True))
loss_cos = tf.clip_by_value(
(Cos_dis(y_oh, logits) - Cos_dis(y_target_oh, logits) + 0.007), 0.0,
2.1)
if FLAGS.loss == "ce":
loss = loss_ce
elif FLAGS.loss == "po":
loss = loss_po
elif FLAGS.loss == "trip_po":
loss = loss_po + 0.01 * loss_cos
# loss += cross_entropy
# loss += -10*Cos_dis(y_target_oh, logits)
noise = -tf.gradients(loss, x)[0]
# TI-
noise = tf.nn.depthwise_conv2d(noise,
stack_kernel,
strides=[1, 1, 1, 1],
padding='SAME')
# CE Cross-entry loss must add this term
if FLAGS.loss == "ce":
noise = noise / tf.reduce_mean(tf.abs(noise), [1, 2, 3],
keep_dims=True)
noise = momentum * grad + noise
x = x + alpha * tf.sign(noise)
x = tf.clip_by_value(x, x_min, x_max)
i = tf.add(i, 1)
return x, y, i, x_max, x_min, noise, y_target, logits
def graph(x, x_adv_v3, y, i, x_max, x_min, grad, pred):
eps = 2.0 * FLAGS.max_epsilon / 255.0
num_iter = FLAGS.num_iter
alpha = eps / num_iter
momentum = FLAGS.momentum
num_classes = 1001
with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
logits_v3, end_points_v3 = inception_v3.inception_v3(
x, num_classes=num_classes, is_training=False)
with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
logits_adv_v3, end_points_adv_v3 = inception_v3.inception_v3(
x,
num_classes=num_classes,
is_training=False,
scope='AdvInceptionV3')
with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
logits_ens3_adv_v3, end_points_ens3_adv_v3 = inception_v3.inception_v3(
x,
num_classes=num_classes,
is_training=False,
scope='Ens3AdvInceptionV3')
with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
logits_ens4_adv_v3, end_points_ens4_adv_v3 = inception_v3.inception_v3(
x,
num_classes=num_classes,
is_training=False,
scope='Ens4AdvInceptionV3')
with slim.arg_scope(inception_v4.inception_v4_arg_scope()):
logits_v4, end_points_v4 = inception_v4.inception_v4(
x, num_classes=num_classes, is_training=False)
with slim.arg_scope(inception_resnet_v2.inception_resnet_v2_arg_scope()):
logits_res_v2, end_points_res_v2 = inception_resnet_v2.inception_resnet_v2(
x, num_classes=num_classes, is_training=False)
with slim.arg_scope(inception_resnet_v2.inception_resnet_v2_arg_scope()):
logits_ensadv_res_v2, end_points_ensadv_res_v2 = inception_resnet_v2.inception_resnet_v2(
x,
num_classes=num_classes,
is_training=False,
scope='EnsAdvInceptionResnetV2')
with slim.arg_scope(resnet_v2.resnet_arg_scope()):
logits_resnet, end_points_resnet = resnet_v2.resnet_v2_101(
x, num_classes=num_classes, is_training=False)
pred = tf.argmax(
end_points_v3['Predictions'] + end_points_adv_v3['Predictions'] + end_points_ens3_adv_v3['Predictions'] + \
end_points_ens4_adv_v3['Predictions'] + end_points_v4['Predictions'] + \
end_points_res_v2['Predictions'] + end_points_ensadv_res_v2['Predictions'] + end_points_resnet['predictions'],
1)
first_round = tf.cast(tf.equal(i, 0), tf.int64)
y = first_round * pred + (1 - first_round) * y
one_hot = tf.one_hot(y, num_classes)
# logits = (logits_v3 + 0.25 * logits_adv_v3 + logits_ens3_adv_v3 + \
# logits_ens4_adv_v3 + logits_v4 + \
# logits_res_v2 + logits_ensadv_res_v2 + logits_resnet) / 7.25
logits = (logits_v3 + logits_ens3_adv_v3 + \
logits_ens4_adv_v3 + logits_v4 + \
logits_res_v2 + logits_ensadv_res_v2 + logits_resnet) / 7
auxlogits = (end_points_v3['AuxLogits'] + 0.25 * end_points_adv_v3['AuxLogits'] + end_points_ens3_adv_v3[
'AuxLogits'] + \
end_points_ens4_adv_v3['AuxLogits'] + end_points_v4['AuxLogits'] + \
end_points_res_v2['AuxLogits'] + end_points_ensadv_res_v2['AuxLogits']) / 6.25
# auxlogits = (end_points_v3['AuxLogits'] + end_points_ens3_adv_v3[
# 'AuxLogits'] + \
# end_points_ens4_adv_v3['AuxLogits'] + end_points_v4['AuxLogits'] + \
# end_points_res_v2['AuxLogits'] + end_points_ensadv_res_v2['AuxLogits']) / 6
cross_entropy = tf.losses.softmax_cross_entropy(one_hot,
logits,
label_smoothing=0.0,
weights=1.0)
cross_entropy += tf.losses.softmax_cross_entropy(one_hot,
auxlogits,
label_smoothing=0.0,
weights=0.4)
noise = tf.gradients(cross_entropy, x)[0]
noise = noise / tf.reduce_mean(tf.abs(noise), [1, 2, 3], keep_dims=True)
noise = momentum * grad + noise
x = x + alpha * tf.sign(noise)
x = tf.clip_by_value(x, x_min, x_max)
i = tf.add(i, 1)
# with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
# logits_adv_v3_one, end_points_adv_v3_one = inception_v3.inception_v3(
# x_adv_v3, num_classes=num_classes, is_training=False, reuse=True, scope='AdvInceptionV3')
# pred_adv_v3 = tf.argmax(end_points_adv_v3_one['Predictions'], axis=1)
return x, x_adv_v3, y, i, x_max, x_min, noise, pred
def main(_):
start = time.clock()
# tf.set_random_seed(1)
# np.random.seed(1)
# Images for inception classifier are normalized to be in [-1, 1] interval,3
# eps is a difference between pixels so it should be in [0, 2] interval.
# Renormalizing epsilon from [0, 255] to [0, 2].
eps = 2.0 * FLAGS.max_epsilon / 255.0
batch_shape = [FLAGS.batch_size, FLAGS.image_height, FLAGS.image_width, 3]
tf.logging.set_verbosity(tf.logging.INFO)
x = tf.placeholder(dtype=tf.float32,
shape=batch_shape) # , name='x_Placeholder'
x_adv_v3 = tf.placeholder(dtype=tf.float32, shape=batch_shape)
x_max = tf.clip_by_value(x + eps, -1.0, 1.0)
x_min = tf.clip_by_value(x - eps, -1.0, 1.0)
y = tf.zeros(shape=FLAGS.batch_size, dtype=tf.int64)
i = tf.zeros(shape=1, dtype=tf.int8)
grad = tf.zeros(shape=batch_shape)
# pred = tf.zeros(shape=FLAGS.batch_size, dtype=tf.int64)
# noise = tf.zeros(shape=[FLAGS.batch_size, 299, 299, 3],dtype=tf.float32)
# cross_entropy = tf.zeros(shape=FLAGS.batch_size,dtype=tf.float32)
with tf.Session() as sess:
eps = 2.0 * FLAGS.max_epsilon / 255.0
num_iter = FLAGS.num_iter
alpha = eps / num_iter
momentum = FLAGS.momentum
num_classes = 1001
# pred = tf.zeros(shape=FLAGS.batch_size, dtype=tf.int64)
# noise = tf.zeros(shape=[FLAGS.batch_size, 299, 299, 3], dtype=tf.float32)
# cross_entropy = tf.zeros(shape=FLAGS.batch_size, dtype=tf.float32)
with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
logits_v3, end_points_v3 = inception_v3.inception_v3(
x, num_classes=num_classes, is_training=False)
with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
logits_adv_v3, end_points_adv_v3 = inception_v3.inception_v3(
x,
num_classes=num_classes,
is_training=False,
scope='AdvInceptionV3')
with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
logits_ens3_adv_v3, end_points_ens3_adv_v3 = inception_v3.inception_v3(
x,
num_classes=num_classes,
is_training=False,
scope='Ens3AdvInceptionV3')
with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
logits_ens4_adv_v3, end_points_ens4_adv_v3 = inception_v3.inception_v3(
x,
num_classes=num_classes,
is_training=False,
scope='Ens4AdvInceptionV3')
with slim.arg_scope(inception_v4.inception_v4_arg_scope()):
logits_v4, end_points_v4 = inception_v4.inception_v4(
x, num_classes=num_classes, is_training=False)
with slim.arg_scope(
inception_resnet_v2.inception_resnet_v2_arg_scope()):
logits_res_v2, end_points_res_v2 = inception_resnet_v2.inception_resnet_v2(
x, num_classes=num_classes, is_training=False)
with slim.arg_scope(
inception_resnet_v2.inception_resnet_v2_arg_scope()):
logits_ensadv_res_v2, end_points_ensadv_res_v2 = inception_resnet_v2.inception_resnet_v2(
x,
num_classes=num_classes,
is_training=False,
scope='EnsAdvInceptionResnetV2')
with slim.arg_scope(resnet_v2.resnet_arg_scope()):
logits_resnet, end_points_resnet = resnet_v2.resnet_v2_101(
x, num_classes=num_classes, is_training=False)
pred_dict = end_points_v3['Predictions'] + end_points_adv_v3['Predictions'] + end_points_ens3_adv_v3['Predictions'] + \
end_points_ens4_adv_v3['Predictions'] + end_points_v4['Predictions'] + \
end_points_res_v2['Predictions'] + end_points_ensadv_res_v2['Predictions'] + end_points_resnet['predictions']
pred = tf.argmax(pred_dict, axis=1)
logits_dict = [
logits_v3, logits_ens3_adv_v3, logits_ens4_adv_v3, logits_v4,
logits_res_v2, logits_ensadv_res_v2, logits_resnet
]
# for j in range(FLAGS.batch_size):
# end_points_v3_Pred = end_points_v3['Predictions'][j][pred[j]]
# # end_points_adv_v3_Pred = end_points_adv_v3['Predictions']
# end_points_ens3_adv_v3_Pred = end_points_ens3_adv_v3['Predictions'][j][pred[j]]
# end_points_ens4_adv_v3_Pred = end_points_ens4_adv_v3['Predictions'][j][pred[j]]
# end_points_v4_Pred = end_points_v4['Predictions'][j][pred[j]]
# end_points_res_v2_Pred = end_points_res_v2['Predictions'][j][pred[j]]
# end_points_ensadv_res_v2_Pred = end_points_ensadv_res_v2['Predictions'][j][pred[j]]
# end_points_resnet_Pred = end_points_resnet['Predictions'][j][pred[j]]
#
#
# ens_Pred_Value = [end_points_v3_Pred * -1, end_points_ens3_adv_v3_Pred * -1,
# end_points_ens4_adv_v3_Pred * -1,
# end_points_v4_Pred * -1, end_points_res_v2_Pred * -1,
# end_points_ensadv_res_v2_Pred * -1,
# end_points_resnet_Pred * -1]
# TopKFit, TopKFitIndx = tf.nn.top_k(ens_Pred_Value, FLAGS.LOW_K)
# # TopKFit = TopKFit * -1
#
# logits_dictionary = tf.placeholder(dtype=tf.int32, shape=(7, FLAGS.batch_size, 1001))
# TopKFitIndex = tf.placeholder(dtype=tf.int32, shape=(FLAGS.batch_size))
# logits[j] = (logits_dictionary[TopKFitIndex[0]] * 5.0 + logits_dictionary[TopKFitIndex[1]] * 4.0 + logits_dictionary[TopKFitIndex[2]] * 3.0 + logits_dictionary[TopKFitIndex[3]] * 2.0 + logits_dictionary[TopKFitIndex[4]] * 1.0) / 15.0
#i_iteration = tf.placeholder(dtype=tf.int32)
#first_round = tf.cast(tf.equal(i_iteration, 0), tf.int64)
#y = first_round * pred
y = tf.placeholder(
dtype=tf.int32, shape=[
2,
]
) # shape= FLAGS.batchsize !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
#y1 = first_round * pred + (1 - first_round) * y
one_hot = tf.one_hot(y, num_classes)
logits_gravity = tf.placeholder(dtype=tf.float32,
shape=[FLAGS.batch_size, 7])
# logits = tf.zeros(shape=[FLAGS.batch_size, 1001], dtype=tf.float32)
#for i in range(2): ########### 2 is the FLAGS.batch_size
logits1 = (logits_dict[0][0] * logits_gravity[0][0] +
logits_dict[1][0] * logits_gravity[0][1] +
logits_dict[2][0] * logits_gravity[0][2] +
logits_dict[3][0] * logits_gravity[0][3] +
logits_dict[4][0] * logits_gravity[0][4] +
logits_dict[5][0] * logits_gravity[0][5] +
logits_dict[6][0] * logits_gravity[0][6]) / 28.0
logits2 = (logits_dict[0][1] * logits_gravity[1][0] +
logits_dict[1][1] * logits_gravity[1][1] +
logits_dict[2][1] * logits_gravity[1][2] +
logits_dict[3][1] * logits_gravity[1][3] +
logits_dict[4][1] * logits_gravity[1][4] +
logits_dict[5][1] * logits_gravity[1][5] +
logits_dict[6][1] * logits_gravity[1][6]) / 28.0
logits1 = tf.reshape(logits1, [1, 1001])
logits2 = tf.reshape(logits2, [1, 1001])
logits = tf.concat([logits1, logits2], 0)
# logits = (logits_v3 + 0.25 * logits_adv_v3 + logits_ens3_adv_v3 + \
# logits_ens4_adv_v3 + logits_v4 + \
# logits_res_v2 + logits_ensadv_res_v2 + logits_resnet) / 7.25
# auxlogits = (end_points_v3['AuxLogits'] + 0.25 * end_points_adv_v3['AuxLogits'] + end_points_ens3_adv_v3['AuxLogits'] + \
# end_points_ens4_adv_v3['AuxLogits'] + end_points_v4['AuxLogits'] + \
# end_points_res_v2['AuxLogits'] + end_points_ensadv_res_v2['AuxLogits']) / 6.25
# cross_entropy = tf.zeros(shape=(FLAGS.batch_size))
#print(str(gravity))
cross_entropy = tf.losses.softmax_cross_entropy(
one_hot,
tf.clip_by_value(logits, 1e-8, tf.reduce_max(logits)),
label_smoothing=0.0,
weights=1.0)
# cross_entropy += tf.losses.softmax_cross_entropy(one_hot,
# auxlogits,
# label_smoothing=0.0,
# weights=0.4)
#print(str(cross_entropy))
noise = tf.gradients(cross_entropy, x)[0]
#print(str(tf.gradients(cross_entropy, x)) + '=========')
# if(noise == None):
# noise = tf.zeros(shape=[FLAGS.batch_size, 299, 299, 3])
# print('noise出了问题！')
noise = noise / tf.reduce_mean(tf.abs(noise), [1, 2, 3],
keep_dims=True)
grad = momentum * grad + noise
adv = x + alpha * tf.sign(grad)
adv = tf.clip_by_value(adv, x_min, x_max)
with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
logits_adv_v3_one, end_points_adv_v3_one = inception_v3.inception_v3(
x_adv_v3,
num_classes=num_classes,
is_training=False,
reuse=True,
scope='AdvInceptionV3')
pred_adv_v3 = tf.argmax(end_points_adv_v3_one['Predictions'], axis=1)
# x_adv, y, _, _, _, _, pred = tf.while_loop(stop, graph, [x, y, i, x_max, x_min, grad, pred])
# Run computation
s1 = tf.train.Saver(slim.get_model_variables(scope='InceptionV3'))
s2 = tf.train.Saver(slim.get_model_variables(scope='AdvInceptionV3'))
s3 = tf.train.Saver(
slim.get_model_variables(scope='Ens3AdvInceptionV3'))
s4 = tf.train.Saver(
slim.get_model_variables(scope='Ens4AdvInceptionV3'))
s5 = tf.train.Saver(slim.get_model_variables(scope='InceptionV4'))
s6 = tf.train.Saver(
slim.get_model_variables(scope='InceptionResnetV2'))
s7 = tf.train.Saver(
slim.get_model_variables(scope='EnsAdvInceptionResnetV2'))
s8 = tf.train.Saver(slim.get_model_variables(scope='resnet_v2'))
s1.restore(sess, FLAGS.checkpoint_path_inception_v3)
s2.restore(sess, FLAGS.checkpoint_path_adv_inception_v3)
s3.restore(sess, FLAGS.checkpoint_path_ens3_adv_inception_v3)
s4.restore(sess, FLAGS.checkpoint_path_ens4_adv_inception_v3)
s5.restore(sess, FLAGS.checkpoint_path_inception_v4)
s6.restore(sess, FLAGS.checkpoint_path_inception_resnet_v2)
s7.restore(sess, FLAGS.checkpoint_path_ens_adv_inception_resnet_v2)
s8.restore(sess, FLAGS.checkpoint_path_resnet)
#sess.run(tf.global_variables_initializer())
#######
# test
# for filenames, images in load_images(FLAGS.input_dir, batch_shape):
# logits_v3_, Pred_dict = sess.run([logits_v3, pred_dict], feed_dict={x: images})
# label = np.argmax(Pred_dict, axis=1)
# print('label::::::', label)
# print('=================\n', logits_v3_)
#######
sum = 0
loss_num = 0
l2_distance = 0
label_distance = 0
for filenames, images in load_images(FLAGS.input_dir, batch_shape):
sum += len(filenames)
images = images.astype(np.float32)
images_flatten_initial = images.reshape((2, 268203))
# true_label = []
# for i in range(len(filenames)):
# true_label.append(filenames[i].split('-')[2][:-4])
#print('image.astype::::::::', images)
#adv_images = tf.zeros(shape=batch_shape)
#adv_images = []
#prediction = 0!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
prediction = []
for i in range(FLAGS.num_iter):
logits_proportion = []
if i == 0:
prediction = sess.run(pred, feed_dict={x: images})
print('true_label::::::::', prediction)
logits_dictionary, end_points_v3_run, end_points_ens3_adv_v3_run, \
end_points_ens4_adv_v3_run, end_points_v4_run, end_points_res_v2_run, end_points_ensadv_res_v2_run, end_points_resnet_run = \
sess.run([logits_dict,end_points_v3, end_points_ens3_adv_v3,
end_points_ens4_adv_v3, end_points_v4, end_points_res_v2, end_points_ensadv_res_v2,
end_points_resnet], feed_dict={x: images, y: prediction})
else:
logits_dictionary, adv_pred, end_points_v3_run, end_points_ens3_adv_v3_run, \
end_points_ens4_adv_v3_run, end_points_v4_run, end_points_res_v2_run, end_points_ensadv_res_v2_run, end_points_resnet_run = \
sess.run([logits_dict, pred, end_points_v3, end_points_ens3_adv_v3,
end_points_ens4_adv_v3, end_points_v4, end_points_res_v2, end_points_ensadv_res_v2,
end_points_resnet], feed_dict={x: images, y: prediction})
# print('prediction::::::::::', prediction)
# print('end_points_v3_run[Predictions]:::::::', end_points_v3_run['Predictions'])
# if i == FLAGS.num_iter - 1:
# print('adv_pred:::::::', adv_pred)
#
# for l in range(len(filenames)):
# label_distance += abs(prediction[l] - adv_pred[l])
#
# if int(prediction[l]) == adv_pred[l]: # The test tag is always 1 more than the real tag, so I subtract 1 here
# loss_num += 1
# #print('sucess_num:::::', loss_num)
for j in range(FLAGS.batch_size):
end_points_v3_Pred = end_points_v3_run['Predictions'][j][
prediction[j]]
# end_points_adv_v3_Pred = end_points_adv_v3['Predictions']
end_points_ens3_adv_v3_Pred = end_points_ens3_adv_v3_run[
'Predictions'][j][prediction[j]]
end_points_ens4_adv_v3_Pred = end_points_ens4_adv_v3_run[
'Predictions'][j][prediction[j]]
end_points_v4_Pred = end_points_v4_run['Predictions'][j][
prediction[j]]
end_points_res_v2_Pred = end_points_res_v2_run[
'Predictions'][j][prediction[j]]
end_points_ensadv_res_v2_Pred = end_points_ensadv_res_v2_run[
'Predictions'][j][prediction[j]]
end_points_resnet_Pred = end_points_resnet_run[
'predictions'][j][prediction[j]]
ens_Pred_Value = [
end_points_v3_Pred, end_points_ens3_adv_v3_Pred,
end_points_ens4_adv_v3_Pred, end_points_v4_Pred,
end_points_res_v2_Pred, end_points_ensadv_res_v2_Pred,
end_points_resnet_Pred
]
TopKFitIndx = np.argsort(ens_Pred_Value)
# logits_dictionary = tf.placeholder(dtype=tf.int32, shape=(7, FLAGS.batch_size, 1001))
# TopKFitIndex = tf.placeholder(dtype=tf.int32, shape=(FLAGS.batch_size))
a = [0.0] * 7
print('ens_Pred_Value:::::::::', ens_Pred_Value)
# print('TopKFitIndx::::::', TopKFitIndx)
# for m in range(7):
# a[TopKFitIndx[m]] = 7 - m
a[TopKFitIndx[0]] = 2
a[TopKFitIndx[1]] = 2
a[TopKFitIndx[2]] = 2
a[TopKFitIndx[3]] = 2
a[TopKFitIndx[4]] = 1
a[TopKFitIndx[5]] = 1
a[TopKFitIndx[6]] = 1
logits_proportion.append(a)
One_hot, Logits_dict, Logits1, Logits2, Logits, images, Cross_Entropy, Noise, gradient = \
sess.run([one_hot, logits_dict, logits1, logits2, logits, adv, cross_entropy, noise, grad],
feed_dict={x: images, logits_gravity: logits_proportion, y: prediction})
if i == FLAGS.num_iter - 1:
images_flatten_adv = images.reshape((2, 268203))
l2_distance_list = np.linalg.norm(
(images_flatten_initial - images_flatten_adv),
axis=1,
keepdims=True)
for n in range(len(filenames)):
l2_distance += l2_distance_list[n]
# print('One_hot::::::::::::', One_hot.shape, One_hot)
# print('Logits_dict::::::::::', Logits_dict)
# print('Logits1::::::::::',Logits1.shape, Logits1)
# print('Logits2::::::::::',Logits2.shape, Logits2)
#
# print('Logits:::::::::::', Logits)
# print('Cross_Entropy:::::::::::::', Cross_Entropy)
# print('Noise:::::::::::', Noise)
# print('gradient::::::::::', gradient)
prediction_adv_v3 = sess.run(pred_adv_v3,
feed_dict={x_adv_v3: images})
print('prediction_adv_v3::::::', prediction_adv_v3)
for l in range(len(filenames)):
label_distance += abs(prediction[l] - prediction_adv_v3[l])
if int(prediction[l]) == prediction_adv_v3[
l]: # The test tag is always 1 more than the real tag, so I subtract 1 here
loss_num += 1
save_images(images, filenames, FLAGS.output_dir)
print('sum::::', sum)
print('loss_num::::', loss_num)
rate_wrong = loss_num / sum
print('rate_wrong:::::::', rate_wrong)
print('l2_distance:::::::', l2_distance)
print('label_distance:::::::', label_distance)
end = time.clock()
print('run time:::::', end - start)
def graph(x, y, i, x_max, x_min, grad, amplification):
one_hot = tf.one_hot(y, FLAGS.num_classes)
eps = 2.0 * FLAGS.max_epsilon / 255.0
num_iter = FLAGS.num_iter
alpha = eps / num_iter
momentum = FLAGS.momentum
# amplification factor
alpha_beta = alpha * FLAGS.amplification_factor
gamma = alpha_beta
# DIM: https://arxiv.org/abs/1803.06978
# input_diversity(FLAG, x)
with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
logits_v3, end_points_v3 = inception_v3.inception_v3(
x, num_classes=FLAGS.num_classes, is_training=False)
auxlogits_v3 = end_points_v3['AuxLogits']
with slim.arg_scope(inception_v4.inception_v4_arg_scope()):
logits_v4, end_points_v4 = inception_v4.inception_v4(
x, num_classes=FLAGS.num_classes, is_training=False)
auxlogits_v4 = end_points_v4['AuxLogits']
with slim.arg_scope(resnet_v2.resnet_arg_scope()):
logits_resnet, end_points_resnet = resnet_v2.resnet_v2_152(
x, num_classes=FLAGS.num_classes, is_training=False)
with slim.arg_scope(inception_resnet_v2.inception_resnet_v2_arg_scope()):
logits_Incres, end_points_IR = inception_resnet_v2.inception_resnet_v2(
x, num_classes=FLAGS.num_classes, is_training=False)
auxlogits_IR = end_points_IR['AuxLogits']
logits = (logits_v3 + logits_v4 + logits_resnet + logits_Incres) / 4.0
auxlogits = (auxlogits_v3 + auxlogits_v4 + auxlogits_IR) / 3.0
cross_entropy = tf.losses.softmax_cross_entropy(one_hot,
logits,
label_smoothing=0.0,
weights=1.0)
cross_entropy += tf.losses.softmax_cross_entropy(one_hot,
auxlogits,
label_smoothing=0.0,
weights=1.0)
noise = tf.gradients(cross_entropy, x)[0]
# TI-FGSM: https://arxiv.org/pdf/1904.02884.pdf
# noise = tf.nn.depthwise_conv2d(noise, T_kern, strides=[1, 1, 1, 1], padding='SAME')
# MI-FGSM: https://arxiv.org/pdf/1710.06081.pdf
# noise = noise / tf.reduce_mean(tf.abs(noise), [1, 2, 3], keep_dims=True)
# noise = momentum * grad + noise
# Project cut noise
amplification += alpha_beta * tf.sign(noise)
cut_noise = tf.clip_by_value(abs(amplification) - eps, 0.0,
10000.0) * tf.sign(amplification)
projection = gamma * tf.sign(project_noise(cut_noise, P_kern, kern_size))
# Occasionally, when the adversarial examples are crafted for an ensemble of networks with residual block by combined methods,
# you may neet to comment the following line to get better result.
amplification += projection
x = x + alpha_beta * tf.sign(noise) + projection
x = tf.clip_by_value(x, x_min, x_max)
i = tf.add(i, 1)
return x, y, i, x_max, x_min, noise, amplification
def graph(x, target_class_input, i, x_max, x_min, grad):
eps = 2.0 * FLAGS.max_epsilon / 255.0
alpha = eps / FLAGS.iterations
momentum = FLAGS.momentum
num_classes = 1001
x_div = input_diversity(x)
#x_div = x
with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
logits_adv_v3, end_points_adv_v3 = inception_v3.inception_v3(
x_div, num_classes=num_classes, is_training=False, scope='AdvInceptionV3')
with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
logits_ens3_adv_v3, end_points_ens3_adv_v3 = inception_v3.inception_v3(
x_div, num_classes=num_classes, is_training=False, scope='Ens3AdvInceptionV3')
with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
logits_ens4_adv_v3, end_points_ens4_adv_v3 = inception_v3.inception_v3(
x_div, num_classes=num_classes, is_training=False, scope='Ens4AdvInceptionV3')
with slim.arg_scope(inception_resnet_v2.inception_resnet_v2_arg_scope()):
logits_ensadv_res_v2, end_points_ensadv_res_v2 = inception_resnet_v2.inception_resnet_v2(
x_div, num_classes=num_classes, is_training=False, scope='EnsAdvInceptionResnetV2')
with slim.arg_scope(inception_resnet_v2.inception_resnet_v2_arg_scope()):
logits_adv_res_v2, end_points_adv_res_v2 = inception_resnet_v2.inception_resnet_v2(
x_div, num_classes=num_classes, is_training=False, scope='AdvInceptionResnetV2')
one_hot_target_class = tf.one_hot(target_class_input, num_classes)
logits = (logits_adv_v3 + logits_ens3_adv_v3 +
logits_ens4_adv_v3 + logits_ensadv_res_v2 + logits_adv_res_v2) / 5
auxlogits = (end_points_adv_v3['AuxLogits'] + end_points_ens3_adv_v3['AuxLogits'] +
end_points_ens4_adv_v3['AuxLogits'] + end_points_adv_res_v2['AuxLogits'] +
end_points_ensadv_res_v2['AuxLogits']) / 5
cross_entropy = tf.losses.softmax_cross_entropy(one_hot_target_class,
logits,
label_smoothing=0.0,
weights=1.0)
cross_entropy += tf.losses.softmax_cross_entropy(one_hot_target_class,
auxlogits,
label_smoothing=0.0,
weights=0.4)
noise = tf.gradients(cross_entropy, x)[0]
kernel = gkern(7, FLAGS.sig).astype(np.float32)
stack_kernel = np.stack([kernel, kernel, kernel]).swapaxes(2, 0)
stack_kernel = np.expand_dims(stack_kernel, 3)
noise = tf.nn.depthwise_conv2d(noise, stack_kernel, strides=[1, 1, 1, 1], padding='SAME')
noise = noise / tf.reshape(tf.contrib.keras.backend.std(tf.reshape(noise, [FLAGS.batch_size, -1]), axis=1),
[FLAGS.batch_size, 1, 1, 1])
noise = momentum * grad + noise
noise = noise / tf.reshape(tf.contrib.keras.backend.std(tf.reshape(noise, [FLAGS.batch_size, -1]), axis=1),
[FLAGS.batch_size, 1, 1, 1])
x = x - alpha * tf.clip_by_value(tf.round(noise), -2, 2)
x = tf.clip_by_value(x, x_min, x_max)
i = tf.add(i, 1)
return x, target_class_input, i, x_max, x_min, noise
},
'inception_v1': {
'shape': [1, 224, 224, 3],
'scope': inception_v1.inception_v1_arg_scope(),
'net': inception_v1.inception_v1,
'num_classes': 1000
},
'inception_v2': {
'shape': [1, 224, 224, 3],
'scope': inception_v2.inception_v2_arg_scope(),
'net': inception_v2.inception_v2,
'num_classes': 1000
},
'inception_v3': {
'shape': [1, 299, 299, 3],
'scope': inception_v3.inception_v3_arg_scope(),
'net': inception_v3.inception_v3,
'num_classes': 1000
},
'vgg_16': {
'shape': [1, 224, 224, 3],
'scope': vgg.vgg_arg_scope(),
'net': vgg.vgg_16,
'num_classes': 1000
},
'vgg_19': {
'shape': [1, 224, 224, 3],
'scope': vgg.vgg_arg_scope(),
'net': vgg.vgg_19,
'num_classes': 1000
},
def graph(x, y, yi, i, x_max, x_min, grad,
pred_w): #[x_input, adv_x, y, i, x_max, x_min, grad]
eps = 2.0 * FLAGS.max_epsilon / 255.0
num_iter = FLAGS.num_iter
alpha = eps / num_iter
momentum = FLAGS.momentum
num_classes = 1001
image = x
with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
logits_inc_v3, end_points_inc_v3 = inception_v3.inception_v3(
image,
num_classes=num_classes,
is_training=False,
scope='InceptionV3')
with slim.arg_scope(inception_v4.inception_v4_arg_scope()):
logits_inc_v4, end_points_inc_v4 = inception_v4.inception_v4(
image,
num_classes=num_classes,
is_training=False,
scope='InceptionV4')
with slim.arg_scope(inception_resnet_v2.inception_resnet_v2_arg_scope()):
logits_inc_res_v2, end_points_inc_res_v2 = inception_resnet_v2.inception_resnet_v2(
image,
num_classes=num_classes,
is_training=False,
scope='InceptionResnetV2')
with slim.arg_scope(inception_resnet_v2.inception_resnet_v2_arg_scope()):
logits_ensadv_res_v2, end_points_ensadv_res_v2 = inception_resnet_v2.inception_resnet_v2(
image,
num_classes=num_classes,
is_training=False,
scope='ens_adv_inception_resnet_v2')
with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
logits_ens3_inc_v3, end_points_ens3_inc_v3 = inception_v3.inception_v3(
image,
num_classes=num_classes,
is_training=False,
scope='ens3_adv_inception_v3')
# with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
# logits_ens4_inc_v3, end_points_ens4_inc_v3 = inception_v3.inception_v3(
# image, num_classes=num_classes, is_training=False, scope='ens4_adv_inception_v3')
pred = tf.argmax(end_points_inc_v3['Predictions']+ \
end_points_inc_v4['Predictions']+end_points_inc_res_v2['Predictions']+ \
end_points_ensadv_res_v2['Predictions']+end_points_ens3_inc_v3['Predictions'], 1)
first_round = tf.cast(tf.equal(i, 0), tf.int64)
y = first_round * pred + (1 - first_round) * y
one_hot = tf.one_hot(y, num_classes)
logits = (logits_inc_v3*pred_w[0]+logits_inc_v4*pred_w[1]+logits_inc_res_v2*pred_w[2] + \
logits_ensadv_res_v2*pred_w[3]+logits_ens3_inc_v3*pred_w[4])/tf.reduce_sum(pred_w)
auxlogits = (end_points_inc_v3['AuxLogits']*pred_w[0]+end_points_inc_v4['AuxLogits']*pred_w[1]+ \
end_points_inc_res_v2['AuxLogits']*pred_w[2]+end_points_ensadv_res_v2['AuxLogits']*pred_w[3]+\
end_points_ens3_inc_v3['AuxLogits']*pred_w[4])/(tf.reduce_sum(pred_w))
logits = logits + auxlogits * 0.4
cross_entropy = tf.losses.softmax_cross_entropy(one_hot,
logits,
label_smoothing=0.0,
weights=1.0)
cross_entropy += tf.losses.softmax_cross_entropy(one_hot,
auxlogits,
label_smoothing=0.0,
weights=0.4)
noise = tf.gradients(cross_entropy, x)[0]
noise = noise / tf.reduce_mean(tf.abs(noise), [1, 2, 3], keep_dims=True)
noise = momentum * grad + noise
x = x + alpha * tf.sign(noise)
x = tf.clip_by_value(x, x_min, x_max)
i = tf.add(i, 1)
inc3_pred = tf.concat([[
end_points_inc_v3['Predictions'][i, yi[i]] for i in range(yi.shape[0])
]],
axis=0)
inc4_pred = tf.concat([[
end_points_inc_v4['Predictions'][i, yi[i]] for i in range(yi.shape[0])
]],
axis=0)
res2_pred = tf.concat([[
end_points_inc_res_v2['Predictions'][i, yi[i]]
for i in range(yi.shape[0])
]],
axis=0)
ens_res2_pred = tf.concat([[
end_points_ensadv_res_v2['Predictions'][i, yi[i]]
for i in range(yi.shape[0])
]],
axis=0)
ens3_inc3_pred = tf.concat([[
end_points_ens3_inc_v3['Predictions'][i, yi[i]]
for i in range(yi.shape[0])
]],
axis=0)
# ens4_inc3_pred = tf.concat([[end_points_ens4_inc_v3['Predictions'][i,yi[i]] for i in range(yi.shape[0])]],axis=0)
pred_list = tf.concat(
[inc3_pred, inc4_pred, res2_pred, ens_res2_pred, ens3_inc3_pred],
axis=0)
return x, y, yi, i, x_max, x_min, noise, pred_list
def run():
# Create the log directory here. Must be done here otherwise import will activate this unneededly.
if not os.path.exists(log_dir):
os.mkdir(log_dir)
# ======================= TRAINING PROCESS =========================
# Now we start to construct the graph and build our model
with tf.Graph().as_default() as graph:
tf.logging.set_verbosity(tf.logging.INFO) # Set the verbosity to INFO level
# First create the dataset and load one batch
dataset = get_split('train', dataset_dir, file_pattern=file_pattern)
images, labels = load_batch(dataset, batch_size=batch_size)
# Know the number steps to take before decaying the learning rate and batches per epoch
num_batches_per_epoch = int(dataset.num_samples / batch_size)
num_steps_per_epoch = num_batches_per_epoch # Because one step is one batch processed
decay_steps = int(num_epochs_before_decay * num_steps_per_epoch)
# Create the model inference
with slim.arg_scope(inception_v3_arg_scope()):
logits, end_points = inception_v3(images, num_classes=dataset.num_classes, is_training=True)
# Define the scopes that you want to exclude for restoration
exclude = ['InceptionV3/Logits', 'InceptionV3/AuxLogits']
variables_to_restore = slim.get_variables_to_restore(exclude=exclude)
# Perform one-hot-encoding of the labels (Try one-hot-encoding within the load_batch function!)
one_hot_labels = slim.one_hot_encoding(labels, dataset.num_classes)
# Performs the equivalent to tf.nn.sparse_softmax_cross_entropy_with_logits but enhanced with checks
loss = tf.losses.softmax_cross_entropy(onehot_labels=one_hot_labels, logits=logits)
total_loss = tf.losses.get_total_loss(add_regularization_losses=True) # obtain the regularization losses as well
# Create the global step for monitoring the learning_rate and training.
global_step = get_or_create_global_step()
# Define your exponentially decaying learning rate
lr = tf.train.exponential_decay(
learning_rate=initial_learning_rate,
global_step=global_step,
decay_steps=decay_steps,
decay_rate=learning_rate_decay_factor,
staircase=True)
# Now we can define the optimizer that takes on the learning rate
optimizer = tf.train.AdamOptimizer(learning_rate=lr)
# Create the train_op.
train_op = slim.learning.create_train_op(total_loss, optimizer)
# State the metrics that you want to predict. We get a predictions that is not one_hot_encoded.
predictions = tf.argmax(end_points['Predictions'], 1)
probabilities = end_points['Predictions']
accuracy, accuracy_update = tf.contrib.metrics.streaming_accuracy(predictions, labels)
metrics_op = tf.group(accuracy_update, probabilities)
# Now finally create all the summaries you need to monitor and group them into one summary op.
tf.summary.scalar('losses/Total_Loss', total_loss)
tf.summary.scalar('accuracy', accuracy)
tf.summary.scalar('learning_rate', lr)
my_summary_op = tf.summary.merge_all()
# Now we need to create a training step function that runs both the train_op, metrics_op and updates the global_step concurrently.
def train_step(sess, train_op, global_step):
'''
Simply runs a session for the three arguments provided and gives a logging on the time elapsed for each global step
'''
# Check the time for each sess run
start_time = time.time()
total_loss, global_step_count, _ = sess.run([train_op, global_step, metrics_op])
time_elapsed = time.time() - start_time
# Run the logging to print some results
logging.info('global step %s: loss: %.4f (%.2f sec/step)', global_step_count, total_loss, time_elapsed)
return total_loss, global_step_count
# Now we create a saver function that actually restores the variables from a checkpoint file in a sess
saver = tf.train.Saver(variables_to_restore)
def restore_fn(sess):
return saver.restore(sess, checkpoint_file)
# Define your supervisor for running a managed session. Do not run the summary_op automatically or else it will consume too much memory
sv = tf.train.Supervisor(logdir=log_dir, summary_op=None, init_fn=restore_fn)
# Run the managed session
with sv.managed_session() as sess:
print (num_steps_per_epoch)
print (num_epochs)
for step in range(num_steps_per_epoch * num_epochs):
# At the start of every epoch, show the vital information:
if step % num_batches_per_epoch == 0:
logging.info('Epoch %s/%s', step / num_batches_per_epoch + 1, num_epochs)
learning_rate_value, accuracy_value = sess.run([lr, accuracy])
logging.info('Current Learning Rate: %s', learning_rate_value)
logging.info('Current Streaming Accuracy: %s', accuracy_value)
# optionally, print your logits and predictions for a sanity check that things are going fine.
logits_value, probabilities_value, predictions_value, labels_value = sess.run(
[logits, probabilities, predictions, labels])
print ('logits: \n', logits_value)
print ('Probabilities: \n', probabilities_value)
print ('predictions: \n', predictions_value)
print ('Labels:\n:', labels_value)
# Log the summaries every 10 step.
if step % 10 == 0:
train_writer = tf.summary.FileWriter('tmp/vis/depth_with_cp')
loss, _ = train_step(sess, train_op, sv.global_step)
summaries = sess.run(my_summary_op)
sv.summary_computed(sess, summaries)
train_writer.add_summary(summaries, step)
train_writer.close()
# If not, simply run the training step
else:
loss, _ = train_step(sess, train_op, sv.global_step)
# We log the final training loss and accuracy
logging.info('Final Loss: %s', loss)
logging.info('Final Accuracy: %s', sess.run(accuracy))
# Once all the training has been done, save the log files and checkpoint model
logging.info('Finished training! Saving model to disk now.')
sv.saver.save(sess, sv.save_path, global_step=sv.global_step)
def graph(x, y, i, x_max, x_min, grad1, grad2, t):
eps = 2 * tf.sqrt(299.0 * 299.0 * 3) * FLAGS.max_epsilon / 255.0
num_iter = FLAGS.num_iter
alpha = eps / num_iter
momentum = FLAGS.momentum
num_classes = 1001
with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
logits_v3, end_points_v3 = inception_v3.inception_v3(
x, num_classes=num_classes, is_training=False)
# with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
# logits_adv_v3, end_points_adv_v3 = inception_v3.inception_v3(
# x, num_classes=num_classes, is_training=False, scope='AdvInceptionV3')
with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
logits_ens3_adv_v3, end_points_ens3_adv_v3 = inception_v3.inception_v3(
x, num_classes=num_classes, is_training=False, scope='Ens3AdvInceptionV3')
with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
logits_ens4_adv_v3, end_points_ens4_adv_v3 = inception_v3.inception_v3(
x, num_classes=num_classes, is_training=False, scope='Ens4AdvInceptionV3')
# with slim.arg_scope(inception_v4.inception_v4_arg_scope()):
# logits_v4, end_points_v4 = inception_v4.inception_v4(
# x, num_classes=num_classes, is_training=False)
with slim.arg_scope(inception_resnet_v2.inception_resnet_v2_arg_scope()):
logits_res_v2, end_points_res_v2 = inception_resnet_v2.inception_resnet_v2(
x, num_classes=num_classes, is_training=False)
with slim.arg_scope(inception_resnet_v2.inception_resnet_v2_arg_scope()):
logits_ensadv_res_v2, end_points_ensadv_res_v2 = inception_resnet_v2.inception_resnet_v2(
x, num_classes=num_classes, is_training=False, scope='EnsAdvInceptionResnetV2')
with slim.arg_scope(resnet_v2.resnet_arg_scope()):
logits_resnet, end_points_resnet = resnet_v2.resnet_v2_101(
x, num_classes=num_classes, is_training=False)
# pred = tf.argmax(end_points_v3['Predictions'] + end_points_adv_v3['Predictions'] + end_points_ens3_adv_v3['Predictions'] + \
# end_points_ens4_adv_v3['Predictions'] + end_points_v4['Predictions'] + \
# end_points_res_v2['Predictions'] + end_points_ensadv_res_v2['Predictions'] + end_points_resnet['predictions'], 1)
pred = tf.argmax(end_points_v3['Predictions'] + end_points_ens3_adv_v3['Predictions'] + \
end_points_ens4_adv_v3['Predictions']+ \
end_points_res_v2['Predictions'] + end_points_ensadv_res_v2['Predictions'] + end_points_resnet['predictions'], 1)
first_round = tf.cast(tf.equal(i, 0), tf.int64)
y = first_round * pred + (1 - first_round) * y
one_hot = tf.one_hot(y, num_classes)
# logits = (logits_v3 + 0.25 * logits_adv_v3 + logits_ens3_adv_v3 + \
# logits_ens4_adv_v3 + logits_v4 + \
# logits_res_v2 + logits_ensadv_res_v2 + logits_resnet) / 7.25
# auxlogits = (end_points_v3['AuxLogits'] + 0.25 * end_points_adv_v3['AuxLogits'] + end_points_ens3_adv_v3['AuxLogits'] + \
# end_points_ens4_adv_v3['AuxLogits'] + end_points_v4['AuxLogits'] + \
# end_points_res_v2['AuxLogits'] + end_points_ensadv_res_v2['AuxLogits']) / 6.25
logits = (logits_v3 + logits_ens3_adv_v3 + \
logits_ens4_adv_v3 + \
logits_res_v2 + logits_ensadv_res_v2 + logits_resnet) / 6.0
auxlogits = (end_points_v3['AuxLogits'] + end_points_ens3_adv_v3['AuxLogits'] + \
end_points_ens4_adv_v3['AuxLogits'] + \
end_points_res_v2['AuxLogits'] + end_points_ensadv_res_v2['AuxLogits']) / 5.0
cross_entropy = tf.losses.softmax_cross_entropy(one_hot,
logits,
label_smoothing=0.0,
weights=1.0)
cross_entropy += tf.losses.softmax_cross_entropy(one_hot,
auxlogits,
label_smoothing=0.0,
weights=0.4)
noise = tf.gradients(cross_entropy, x)[0]
b = tf.random_normal(shape=grad1.shape, mean=0, stddev=1.0)
prob = 0.4 / (100 * ((i + 1) / FLAGS.num_iter))
b1 = tf.nn.dropout(b, keep_prob=prob)
b2 = tf.sign(b1)
b3 = 0.5 * tf.ones(shape=grad1.shape) + b2
b4 = tf.sign(b3)
noise = (noise / tf.reduce_mean(tf.abs(noise), [1, 2, 3], keepdims=True))
grad = momentum * grad2 + noise + (1.0 - (1.0 / FLAGS.num_iter) * i) * momentum * (grad2 - grad1)
grad1 = grad2
grad2 = grad
grad3 = grad * b4
noise_norm = tf.norm(grad3, axis=(1, 2), ord='fro', keepdims=True)
x = x + alpha * (grad3 / noise_norm)
x = tf.clip_by_value(x, x_min, x_max)
i = tf.add(i, 1)
return x, y, i, x_max, x_min, grad1, grad2, t
def run():
end_points = {}
# Create the log directory here. Must be done here otherwise import will activate this unneededly.
if not os.path.exists(log_dir):
os.mkdir(log_dir)
# ======================= TRAINING PROCESS =========================
# Now we start to construct the graph and build our model
with tf.Graph().as_default() as graph:
tf.logging.set_verbosity(
tf.logging.INFO) # Set the verbosity to INFO level
########################################################
# Get RGB dataset and the Imagenet trained on RGB images
########################################################
# First create the dataset and load one batch
dataset_rgb = get_split('train',
dataset_dir_rgb,
file_pattern=file_pattern)
images_rgb, labels_rgb = load_batch(dataset_rgb, batch_size=batch_size)
num_batches_per_epoch = int(dataset_rgb.num_samples / batch_size)
num_steps_per_epoch = num_batches_per_epoch # Because one step is one batch processed
decay_steps = int(num_epochs_before_decay * num_steps_per_epoch)
with tf.variable_scope("net_rgb"):
# Create the model inference
with slim.arg_scope(inception_v3_arg_scope()):
logits_rgb, end_points_rgb = inception_v3(
images_rgb,
num_classes=dataset_rgb.num_classes,
is_training=True)
net1_varlist = {
v.name.lstrip("net_rgb/")[:-2]: v
for v in tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES,
scope="net_rgb/")
}
########################################################
# Get depth dataset and the Imagenet trained on depth images
########################################################
# First create the dataset and load one batch
dataset_depth = get_split('train',
dataset_dir_depth,
file_pattern=file_pattern)
images_depth, labels_depth = load_batch(dataset_depth,
batch_size=batch_size)
num_batches_per_epoch = int(dataset_rgb.num_samples / batch_size)
num_steps_per_epoch = num_batches_per_epoch # Because one step is one batch processed
decay_steps = int(num_epochs_before_decay * num_steps_per_epoch)
# Create the model inference
with tf.variable_scope("net_depth"):
with slim.arg_scope(inception_v3_arg_scope()):
logits_depth, end_points_depth = inception_v3(
images_depth,
num_classes=dataset_rgb.num_classes,
is_training=True)
net2_varlist = {
v.name.lstrip("net_depth/")[:-2]: v
for v in tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES,
scope="net_depth/")
}
########################################################
# Combine the models with the concatenation operation
# and add an FC layer on top
########################################################
#
with tf.variable_scope("concat_dense"):
W_master = tf.Variable(tf.random_uniform([10, 5], -0.01, 0.01),
name="weights_concat")
b_master = tf.Variable(tf.zeros([5]), name="bias_concat")
h_master = tf.matmul(tf.concat(
(logits_rgb, logits_depth), axis=1), W_master) + b_master
variable_summaries(W_master, "concat")
logits2 = tf.layers.dense(inputs=h_master,
units=(num_classes * 2),
name="dense_concat1")
logits = tf.layers.dense(inputs=logits2,
units=num_classes,
name="dense_concat0")
end_points['Logits'] = logits
end_points['Predictions'] = slim.softmax(logits, scope='Predictions')
one_hot_labels = slim.one_hot_encoding(labels_rgb,
dataset_rgb.num_classes)
loss = tf.losses.softmax_cross_entropy(onehot_labels=one_hot_labels,
logits=logits)
tf.summary.scalar('cross_entropy', loss)
total_loss = tf.losses.get_total_loss(add_regularization_losses=True)
global_step = get_or_create_global_step()
lr = tf.train.exponential_decay(learning_rate=initial_learning_rate,
global_step=global_step,
decay_steps=decay_steps,
decay_rate=learning_rate_decay_factor,
staircase=True)
optimizer = tf.train.AdamOptimizer(learning_rate=lr)
train_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES,
"concat_dense")
train_op = slim.learning.create_train_op(total_loss,
optimizer,
variables_to_train=train_vars)
# Metrics
predictions = tf.argmax(end_points['Predictions'], 1)
probabilities = end_points['Predictions']
accuracy, accuracy_update = tf.contrib.metrics.streaming_accuracy(
predictions, labels_rgb)
metrics_op = tf.group(accuracy_update, probabilities)
tf.summary.scalar('losses/Total_Loss', total_loss)
tf.summary.scalar('accuracy', accuracy)
tf.summary.scalar('learning_rate', lr)
my_summary_op = tf.summary.merge_all()
def train_step(sess, train_op, global_step):
'''
Simply runs a session for the three arguments provided and gives a logging on the time elapsed for each global step
'''
# Check the time for each sess run
start_time = time.time()
total_loss, global_step_count, _ = sess.run(
[train_op, global_step, metrics_op])
time_elapsed = time.time() - start_time
# Run the logging to print some results
logging.info('global step %s: loss: %.4f (%.2f sec/step)',
global_step_count, total_loss, time_elapsed)
return total_loss, global_step_count
saver_rgb = tf.train.Saver(var_list=net1_varlist)
saver_depth = tf.train.Saver(var_list=net2_varlist)
sv = tf.train.Supervisor(logdir=log_dir, summary_op=None, init_fn=None)
with sv.managed_session() as sess:
print(num_steps_per_epoch)
print(num_epochs)
rgb_latest = tf.train.latest_checkpoint(
os.path.join(os.getcwd(), 'tmp/log/rgb_no_cp'))
depth_latest = tf.train.latest_checkpoint(
os.path.join(os.getcwd(), 'tmp/log/depth_no_cp'))
saver_rgb.restore(sess, rgb_latest)
saver_depth.restore(sess, depth_latest)
for step in range(num_steps_per_epoch * num_epochs):
if step % num_batches_per_epoch == 0:
logging.info('Epoch %s/%s',
step / num_batches_per_epoch + 1, num_epochs)
learning_rate_value, accuracy_value = sess.run(
[lr, accuracy])
logging.info('Current Learning Rate: %s',
learning_rate_value)
logging.info('Current Streaming Accuracy: %s',
accuracy_value)
if step % 10 == 0:
train_writer = tf.summary.FileWriter(
'tmp/vis/both_no_cp', sess.graph)
loss, _ = train_step(sess, train_op, sv.global_step)
summaries = sess.run(my_summary_op)
sv.summary_computed(sess, summaries)
train_writer.add_summary(summaries, step)
train_writer.close()
# If not, simply run the training step
else:
loss, _ = train_step(sess, train_op, sv.global_step)
logging.info('Final Loss: %s', loss)
logging.info('Final Accuracy: %s', sess.run(accuracy))
logging.info('Finished training! Saving model to disk now.')
sv.saver.save(sess, sv.save_path, global_step=sv.global_step)
def Evaluator(x, y):
num_classes = 1001
# should keep original x here for output
with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
logits_v3, end_points_v3 = inception_v3.inception_v3(
x, num_classes=num_classes, is_training=False)
with slim.arg_scope(inception_v4.inception_v4_arg_scope()):
logits_v4, end_points_v4 = inception_v4.inception_v4(
x, num_classes=num_classes, is_training=False)
with slim.arg_scope(inception_resnet_v2.inception_resnet_v2_arg_scope()):
logits_res_v2, end_points_res_v2 = inception_resnet_v2.inception_resnet_v2(
x, num_classes=num_classes, is_training=False, reuse=True)
with slim.arg_scope(resnet_v2.resnet_arg_scope()):
logits_resnet, end_points_resnet = resnet_v2.resnet_v2_152(
x, num_classes=num_classes, is_training=False)
with slim.arg_scope(resnet_v2.resnet_arg_scope()):
logits_resnet_101, end_points_resnet_101 = resnet_v2.resnet_v2_101(
x, num_classes=num_classes, is_training=False, reuse=tf.AUTO_REUSE)
with slim.arg_scope(resnet_v2.resnet_arg_scope()):
logits_resnet_50, end_points_resnet_50 = resnet_v2.resnet_v2_50(
x, num_classes=num_classes, is_training=False, reuse=tf.AUTO_REUSE)
with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
logits_adv_v3, end_points_adv_v3 = inception_v3.inception_v3(
x,
num_classes=num_classes,
is_training=False,
scope='AdvInceptionV3')
with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
logits_ens3_adv_v3, end_points_ens3_adv_v3 = inception_v3.inception_v3(
x,
num_classes=num_classes,
is_training=False,
scope='Ens3AdvInceptionV3')
with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
logits_ens4_adv_v3, end_points_ens4_adv_v3 = inception_v3.inception_v3(
x,
num_classes=num_classes,
is_training=False,
scope='Ens4AdvInceptionV3')
with slim.arg_scope(inception_resnet_v2.inception_resnet_v2_arg_scope()):
logits_ensadv_res_v2, end_points_ensadv_res_v2 = inception_resnet_v2.inception_resnet_v2(
x,
num_classes=num_classes,
is_training=False,
scope='EnsAdvInceptionResnetV2')
# with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
# logits_ens_v3, end_points_ens_v3 = inception_v3.inception_v3(
# x, num_classes=num_classes, is_training=False)
# acc_v3 = tf.reduce_sum(tf.cast(tf.equal(tf.argmax(end_points_v3['Predictions'], 1), y), tf.float32))
# acc_v4 = tf.reduce_sum(tf.cast(tf.equal(tf.argmax(end_points_v4['Predictions'], 1), y), tf.float32))
# acc_res_v2 = tf.reduce_sum(tf.cast(tf.equal(tf.argmax(end_points_res_v2['Predictions'], 1), y), tf.float32))
# acc_resnet = tf.reduce_sum(tf.cast(tf.equal(tf.argmax(end_points_resnet['predictions'], 1), y), tf.float32))
# acc_resnet_50 = tf.reduce_sum(tf.cast(tf.equal(tf.argmax(end_points_resnet_50['predictions'], 1), y), tf.float32))
# acc_resnet_101 = tf.reduce_sum(tf.cast(tf.equal(tf.argmax(end_points_resnet_101['predictions'], 1), y), tf.float32))
# acc_ens_v3 = tf.reduce_sum(tf.cast(tf.equal(tf.argmax(end_points_ens_v3['predictions'], 1), y), tf.float32))
logits_esm = (logits_v3 + logits_v4 + logits_res_v2 + logits_resnet +
logits_resnet_50 + logits_resnet_101)
if FLAGS.att_model == "incep_v3":
logits_esm = (logits_esm - logits_v3) / 5
elif FLAGS.att_model == "incep_v4":
logits_esm = (logits_esm - logits_v4) / 5
elif FLAGS.att_model == "incep_res_v2":
logits_esm = (logits_esm - logits_res_v2) / 5
elif FLAGS.att_model == "resnet_50":
logits_esm = (logits_esm - logits_resnet_50) / 5
elif FLAGS.att_model == "resnet_101":
logits_esm = (logits_esm - logits_resnet_101) / 5
elif FLAGS.att_model == "resnet_152":
logits_esm = (logits_esm - logits_resnet) / 5
elif FLAGS.att_model == "ens3_adv_3":
logits_esm = (logits_esm + logits_ens4_adv_v3 +
logits_ensadv_res_v2) / 8
elif FLAGS.att_model == "ens4_adv_3":
logits_esm = (logits_esm + logits_ens3_adv_v3 +
logits_ensadv_res_v2) / 8
elif FLAGS.att_model == "ensadv_res_2":
logits_esm = (logits_esm + logits_ens4_adv_v3 + logits_ens3_adv_v3) / 8
# top_k
top_k = FLAGS.top_k
acc_v3 = tf.reduce_sum(
tf.cast(tf.nn.in_top_k(end_points_v3['Predictions'], y, k=top_k),
tf.float32))
acc_v4 = tf.reduce_sum(
tf.cast(tf.nn.in_top_k(end_points_v4['Predictions'], y, k=top_k),
tf.float32))
acc_res_v2 = tf.reduce_sum(
tf.cast(tf.nn.in_top_k(end_points_res_v2['Predictions'], y, k=top_k),
tf.float32))
acc_resnet = tf.reduce_sum(
tf.cast(tf.nn.in_top_k(end_points_resnet['predictions'], y, k=top_k),
tf.float32))
acc_resnet_50 = tf.reduce_sum(
tf.cast(
tf.nn.in_top_k(end_points_resnet_50['predictions'], y, k=top_k),
tf.float32))
acc_resnet_101 = tf.reduce_sum(
tf.cast(
tf.nn.in_top_k(end_points_resnet_101['predictions'], y, k=top_k),
tf.float32))
acc_adv_v3 = tf.reduce_sum(
tf.cast(tf.nn.in_top_k(end_points_adv_v3['Predictions'], y, k=top_k),
tf.float32))
acc_ens3_adv_v3 = tf.reduce_sum(
tf.cast(
tf.nn.in_top_k(end_points_ens3_adv_v3['Predictions'], y, k=top_k),
tf.float32))
acc_ens4_adv_v3 = tf.reduce_sum(
tf.cast(
tf.nn.in_top_k(end_points_ens4_adv_v3['Predictions'], y, k=top_k),
tf.float32))
acc_ensadv_res_v2 = tf.reduce_sum(
tf.cast(
tf.nn.in_top_k(end_points_ensadv_res_v2['Predictions'], y,
k=top_k), tf.float32))
acc_esm = tf.reduce_sum(
tf.cast(
tf.nn.in_top_k(slim.softmax(logits_esm, scope='predictions'),
y,
k=top_k), tf.float32))
# acc_ens_v3 = tf.reduce_sum(tf.cast(tf.equal(tf.argmax(end_points_ens_v3['predictions'], 1), y), tf.float32))
# pred1 = tf.argmax(end_points_v3['Predictions'], 1)
# pred2 = tf.argmax(end_points_v4['Predictions'], 1)
# pred3 = tf.argmax(end_points_res_v2['Predictions'], 1)
# pred4 = tf.argmax(end_points_resnet['predictions'], 1)
return acc_v3, acc_v4, acc_res_v2, acc_resnet, acc_resnet_50, acc_resnet_101, acc_adv_v3, acc_ens3_adv_v3, \
acc_ens4_adv_v3, acc_ensadv_res_v2, acc_esm, end_points_v3, end_points_v4, end_points_res_v2, end_points_resnet
def Evaluator(x, y):
num_classes = 1001
# should keep original x here for output
with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
logits_v3, end_points_v3 = inception_v3.inception_v3(
x, num_classes=num_classes, is_training=False)
with slim.arg_scope(inception_v4.inception_v4_arg_scope()):
logits_v4, end_points_v4 = inception_v4.inception_v4(
x, num_classes=num_classes, is_training=False)
with slim.arg_scope(inception_resnet_v2.inception_resnet_v2_arg_scope()):
logits_res_v2, end_points_res_v2 = inception_resnet_v2.inception_resnet_v2(
x, num_classes=num_classes, is_training=False, reuse=True)
with slim.arg_scope(resnet_v2.resnet_arg_scope()):
logits_resnet, end_points_resnet = resnet_v2.resnet_v2_152(
x, num_classes=num_classes, is_training=False)
with slim.arg_scope(resnet_v2.resnet_arg_scope()):
logits_resnet_101, end_points_resnet_101 = resnet_v2.resnet_v2_101(
x, num_classes=num_classes, is_training=False, reuse=tf.AUTO_REUSE)
with slim.arg_scope(resnet_v2.resnet_arg_scope()):
logits_resnet_50, end_points_resnet_50 = resnet_v2.resnet_v2_50(
x, num_classes=num_classes, is_training=False, reuse=tf.AUTO_REUSE)
# with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
# logits_adv_v3, end_points_adv_v3 = inception_v3.inception_v3(
# x, num_classes=num_classes, is_training=False, scope='AdvInceptionV3')
#
# with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
# logits_ens3_adv_v3, end_points_ens3_adv_v3 = inception_v3.inception_v3(
# x, num_classes=num_classes, is_training=False, scope='Ens3AdvInceptionV3')
#
# with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
# logits_ens4_adv_v3, end_points_ens4_adv_v3 = inception_v3.inception_v3(
# x, num_classes=num_classes, is_training=False, scope='Ens4AdvInceptionV3')
#
# with slim.arg_scope(inception_resnet_v2.inception_resnet_v2_arg_scope()):
# logits_ensadv_res_v2, end_points_ensadv_res_v2 = inception_resnet_v2.inception_resnet_v2(
# x, num_classes=num_classes, is_training=False, scope='EnsAdvInceptionResnetV2')
# with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
# logits_ens_v3, end_points_ens_v3 = inception_v3.inception_v3(
# x, num_classes=num_classes, is_training=False)
# acc_v3 = tf.reduce_sum(tf.cast(tf.equal(tf.argmax(end_points_v3['Predictions'], 1), y), tf.float32))
# acc_v4 = tf.reduce_sum(tf.cast(tf.equal(tf.argmax(end_points_v4['Predictions'], 1), y), tf.float32))
# acc_res_v2 = tf.reduce_sum(tf.cast(tf.equal(tf.argmax(end_points_res_v2['Predictions'], 1), y), tf.float32))
# acc_resnet = tf.reduce_sum(tf.cast(tf.equal(tf.argmax(end_points_resnet['predictions'], 1), y), tf.float32))
# acc_resnet_50 = tf.reduce_sum(tf.cast(tf.equal(tf.argmax(end_points_resnet_50['predictions'], 1), y), tf.float32))
# acc_resnet_101 = tf.reduce_sum(tf.cast(tf.equal(tf.argmax(end_points_resnet_101['predictions'], 1), y), tf.float32))
# acc_ens_v3 = tf.reduce_sum(tf.cast(tf.equal(tf.argmax(end_points_ens_v3['predictions'], 1), y), tf.float32))
# top_k
top_k = FLAGS.top_k
acc_v3 = tf.reduce_sum(
tf.cast(tf.nn.in_top_k(end_points_v3['Predictions'], y, k=top_k),
tf.float32))
acc_v4 = tf.reduce_sum(
tf.cast(tf.nn.in_top_k(end_points_v4['Predictions'], y, k=top_k),
tf.float32))
acc_res_v2 = tf.reduce_sum(
tf.cast(tf.nn.in_top_k(end_points_res_v2['Predictions'], y, k=top_k),
tf.float32))
acc_resnet = tf.reduce_sum(
tf.cast(tf.nn.in_top_k(end_points_resnet['predictions'], y, k=top_k),
tf.float32))
acc_resnet_50 = tf.reduce_sum(
tf.cast(
tf.nn.in_top_k(end_points_resnet_50['predictions'], y, k=top_k),
tf.float32))
acc_resnet_101 = tf.reduce_sum(
tf.cast(
tf.nn.in_top_k(end_points_resnet_101['predictions'], y, k=top_k),
tf.float32))
# acc_adv_v3 = tf.reduce_sum(tf.cast(tf.nn.in_top_k(end_points_adv_v3['Predictions'], y, k=top_k), tf.float32))
# acc_ens3_adv_v3 = tf.reduce_sum(tf.cast(tf.nn.in_top_k(end_points_ens3_adv_v3['Predictions'], y, k=top_k), tf.float32))
# acc_ens4_adv_v3 = tf.reduce_sum(tf.cast(tf.nn.in_top_k(end_points_ens4_adv_v3['Predictions'], y, k=top_k), tf.float32))
# acc_ensadv_res_v2 = tf.reduce_sum(tf.cast(tf.nn.in_top_k(end_points_ensadv_res_v2['Predictions'], y, k=top_k), tf.float32))
# acc_ens_v3 = tf.reduce_sum(tf.cast(tf.equal(tf.argmax(end_points_ens_v3['predictions'], 1), y), tf.float32))
# pred1 = tf.argmax(end_points_v3['Predictions'], 1)
# pred2 = tf.argmax(end_points_v4['Predictions'], 1)
# pred3 = tf.argmax(end_points_res_v2['Predictions'], 1)
# pred4 = tf.argmax(end_points_resnet['predictions'], 1)
return acc_v3, acc_v4, acc_res_v2, acc_resnet, acc_resnet_50, acc_resnet_101, end_points_v3, end_points_v4, end_points_res_v2, end_points_resnet
def run():
if not os.path.exists(log_eval):
os.mkdir(log_eval)
with tf.Graph().as_default() as graph:
tf.logging.set_verbosity(tf.logging.INFO)
dataset = get_split('validation', dataset_dir)
images, labels = load_batch(dataset,
batch_size=batch_size,
is_training=True)
num_batches_per_epoch = dataset.num_samples / batch_size
num_steps_per_epoch = num_batches_per_epoch
with slim.arg_scope(inception_v3_arg_scope()):
logits, end_points = inception_v3(images,
num_classes=dataset.num_classes,
is_training=True)
variables_to_restore = slim.get_variables_to_restore()
saver = tf.train.Saver(variables_to_restore)
def restore_fn(sess):
return saver.restore(sess, checkpoint_file)
predictions = tf.argmax(end_points['Predictions'], 1)
accuracy, accuracy_update = tf.contrib.metrics.streaming_accuracy(
predictions, labels)
acc, acc_op = tf.metrics.accuracy(labels=labels,
predictions=predictions)
metrics_op = tf.group(accuracy_update)
global_step = get_or_create_global_step()
global_step_op = tf.assign(global_step, global_step + 1)
def eval_step(sess, metrics_op, global_step):
'''
Simply takes in a session, runs the metrics op and some logging information.
'''
start_time = time.time()
_, global_step_count, accuracy_value = sess.run(
[metrics_op, global_step_op, accuracy])
time_elapsed = time.time() - start_time
logging.info(
'Global Step %s: Streaming Accuracy: %.4f (%.2f sec/step)',
global_step_count, accuracy_value, time_elapsed)
return accuracy_value
tf.summary.scalar('Validation_Accuracy', accuracy)
my_summary_op = tf.summary.merge_all()
sv = tf.train.Supervisor(logdir=log_eval,
summary_op=None,
saver=None,
init_fn=restore_fn)
with sv.managed_session() as sess:
num_steps_per_epoch = int(num_steps_per_epoch)
for step in range(num_steps_per_epoch * num_epochs):
sess.run(sv.global_step)
if step % num_batches_per_epoch == 0:
logging.info('Epoch: %s/%s',
step / num_batches_per_epoch + 1, num_epochs)
logging.info('Current Streaming Accuracy: %.4f',
sess.run(accuracy))
if step % 10 == 0:
eval_step(sess,
metrics_op=metrics_op,
global_step=sv.global_step)
summaries = sess.run(my_summary_op)
sv.summary_computed(sess, summaries)
else:
eval_step(sess,
metrics_op=metrics_op,
global_step=sv.global_step)
logging.info('Final Streaming Accuracy: %.4f', sess.run(accuracy))
conf_m = confusion_matrix(labels, predictions)
print(conf_m)
for i in range(10):
image, label, prediction = images[i], labels[i], predictions[i]
prediction_name, label_name = dataset.labels_to_name[
prediction], dataset.labels_to_name[label]
text = 'Prediction: %s \n Ground Truth: %s' % (prediction_name,
label_name)
print(text)
logging.info(
'Model evaluation has completed! Visit TensorBoard for more information regarding your evaluation.'
)
def graph(x, target_class_input, true_label_input, i, x_max, x_min, grad):
eps = 2.0 * FLAGS.max_epsilon / 255.0
alpha = eps / FLAGS.iterations
momentum = FLAGS.momentum
num_classes = 1001
x_div = input_diversity(x)
with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
logits_ens3_adv_v3, end_points_ens3_adv_v3 = inception_v3.inception_v3(
x_div, num_classes=num_classes, is_training=False, scope='Ens3AdvInceptionV3')
with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
logits_ens4_adv_v3, end_points_ens4_adv_v3 = inception_v3.inception_v3(
x_div, num_classes=num_classes, is_training=False, scope='Ens4AdvInceptionV3')
with slim.arg_scope(inception_resnet_v2.inception_resnet_v2_arg_scope()):
logits_ensadv_res_v2, end_points_ensadv_res_v2 = inception_resnet_v2.inception_resnet_v2(
x_div, num_classes=num_classes, is_training=False, scope='EnsAdvInceptionResnetV2')
with slim.arg_scope(inception_resnet_v2.inception_resnet_v2_arg_scope()):
logits_adv_res_v2, end_points_adv_res_v2 = inception_resnet_v2.inception_resnet_v2(
x_div, num_classes=num_classes, is_training=False, scope='AdvInceptionResnetV2')
with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
logits_inc_v3, end_points_inc_v3 = inception_v3.inception_v3(
x_div, num_classes=num_classes, is_training=False)
with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
logits_adv_v3, end_points_adv_v3 = inception_v3.inception_v3(
x_div, num_classes=num_classes, is_training=False, scope='AdvInceptionV3')
with slim.arg_scope(resnet_v2.resnet_arg_scope()):
logits_res_v2, end_points_res_v2 = resnet_v2.resnet_v2_50(
x_div, num_classes=num_classes, is_training=False, scope='resnet_v2_50')
with slim.arg_scope(resnet_v2.resnet_arg_scope()):
logits_res_v2_101, end_points_res_v2_101 = resnet_v2.resnet_v2_101(
x_div, num_classes=num_classes, is_training=False, scope='resnet_v2_101')
with TowerContext(tower_name='eval', is_training=False):
logits_advresnext_101 = resnext.ResNeXtDenoiseAllModel().get_logits(preprocess_for_model(x_div))
one_hot_target_class = tf.one_hot(target_class_input, num_classes)
one_hot_true_label = tf.one_hot(true_label_input, num_classes)
one_hot_target_class_resnext = tf.one_hot(target_class_input-1, num_classes-1)
one_hot_true_label_resnext = tf.one_hot(true_label_input-1, num_classes-1)
logits = (logits_inc_v3 + logits_ens3_adv_v3 +
logits_ens4_adv_v3 + logits_ensadv_res_v2 + logits_adv_res_v2 +logits_adv_v3+logits_res_v2+logits_res_v2_101) / 8
auxlogits = (end_points_inc_v3['AuxLogits'] + end_points_ens3_adv_v3['AuxLogits'] +
end_points_ens4_adv_v3['AuxLogits'] + end_points_adv_res_v2['AuxLogits'] +
end_points_ensadv_res_v2['AuxLogits'] +end_points_adv_v3['AuxLogits']+logits_res_v2+logits_res_v2_101) / 8
logits_resnext = logits_advresnext_101
cross_entropy = -tf.losses.softmax_cross_entropy(one_hot_true_label,
logits,
label_smoothing=0.0,
weights=1.0)
cross_entropy -= tf.losses.softmax_cross_entropy(one_hot_true_label,
auxlogits,
label_smoothing=0.0,
weights=0.4)
cross_entropy2 = tf.losses.softmax_cross_entropy(one_hot_true_label_resnext,
logits_resnext,
label_smoothing=0.0,
weights=1.0)
'''cross_entropy -= tf.losses.softmax_cross_entropy(one_hot_true_label,
logits_ensadv_res_v2,
label_smoothing=0.0,
weights=1.0)
cross_entropy -= tf.losses.softmax_cross_entropy(one_hot_true_label,
end_points_ensadv_res_v2['AuxLogits'],
label_smoothing=0.0,
weights=0.4)'''
rnd = i%2
cross_entropy = tf.cond(tf.greater(rnd,0),lambda:cross_entropy,lambda:cross_entropy2)
noise = tf.gradients(cross_entropy, x)[0]
kernel = gkern(7, FLAGS.sig).astype(np.float32)
#kernel = gkern(15, 4).astype(np.float32)
stack_kernel = np.stack([kernel, kernel, kernel]).swapaxes(2, 0)
stack_kernel = np.expand_dims(stack_kernel, 3)
noise = tf.nn.depthwise_conv2d(noise, stack_kernel, strides=[1, 1, 1, 1], padding='SAME')
noise = noise / tf.reshape(tf.contrib.keras.backend.std(tf.reshape(noise, [FLAGS.batch_size, -1]), axis=1),
[FLAGS.batch_size, 1, 1, 1])
noise = momentum * grad + noise
noise = noise / tf.reshape(tf.contrib.keras.backend.std(tf.reshape(noise, [FLAGS.batch_size, -1]), axis=1),
[FLAGS.batch_size, 1, 1, 1])
x = x - alpha * tf.clip_by_value(tf.round(noise), -2, 2)
x = tf.clip_by_value(x, x_min, x_max)
i = tf.add(i, 1)
return x, target_class_input, true_label_input, i, x_max, x_min, noise
def main(_):
start = time.clock()
# Images for inception classifier are normalized to be in [-1, 1] interval,
# eps is a difference between pixels so it should be in [0, 2] interval.
# Renormalizing epsilon from [0, 255] to [0, 2].
eps = 2.0 * FLAGS.max_epsilon / 255.0
batch_shape = [FLAGS.batch_size, FLAGS.image_height, FLAGS.image_width, 3]
tf.logging.set_verbosity(tf.logging.INFO)
with tf.Graph().as_default():
# Prepare graph
x_input = tf.placeholder(tf.float32, shape=batch_shape)
x_adv_v3 = tf.placeholder(tf.float32, shape=batch_shape)
x_max = tf.clip_by_value(x_input + eps, -1.0, 1.0)
x_min = tf.clip_by_value(x_input - eps, -1.0, 1.0)
y = tf.constant(np.zeros([FLAGS.batch_size]), tf.int64)
i = tf.constant(0)
grad = tf.zeros(shape=batch_shape)
pred = tf.zeros(shape=[
2,
], dtype=tf.int64)
Prediction_adv_v3 = tf.zeros(shape=[
2,
], dtype=tf.int64)
x_adv, _, label, _, _, _, _, Pred = tf.while_loop(
stop, graph, [x_input, x_adv_v3, y, i, x_max, x_min, grad, pred])
with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
logits_adv_v3_one, end_points_adv_v3_one = inception_v3.inception_v3(
x_adv_v3,
num_classes=1001,
is_training=False,
reuse=True,
scope='AdvInceptionV3')
pred_adv_v3 = tf.argmax(end_points_adv_v3_one['Predictions'], axis=1)
# Run computation
s1 = tf.train.Saver(slim.get_model_variables(scope='InceptionV3'))
s2 = tf.train.Saver(slim.get_model_variables(scope='AdvInceptionV3'))
s3 = tf.train.Saver(
slim.get_model_variables(scope='Ens3AdvInceptionV3'))
s4 = tf.train.Saver(
slim.get_model_variables(scope='Ens4AdvInceptionV3'))
s5 = tf.train.Saver(slim.get_model_variables(scope='InceptionV4'))
s6 = tf.train.Saver(
slim.get_model_variables(scope='InceptionResnetV2'))
s7 = tf.train.Saver(
slim.get_model_variables(scope='EnsAdvInceptionResnetV2'))
s8 = tf.train.Saver(slim.get_model_variables(scope='resnet_v2'))
with tf.Session() as sess:
s1.restore(sess, FLAGS.checkpoint_path_inception_v3)
s2.restore(sess, FLAGS.checkpoint_path_adv_inception_v3)
s3.restore(sess, FLAGS.checkpoint_path_ens3_adv_inception_v3)
s4.restore(sess, FLAGS.checkpoint_path_ens4_adv_inception_v3)
s5.restore(sess, FLAGS.checkpoint_path_inception_v4)
s6.restore(sess, FLAGS.checkpoint_path_inception_resnet_v2)
s7.restore(sess, FLAGS.checkpoint_path_ens_adv_inception_resnet_v2)
s8.restore(sess, FLAGS.checkpoint_path_resnet)
sum = 0
loss_num = 0
l2_distance = 0
label_distance = 0
for filenames, images in load_images(FLAGS.input_dir, batch_shape):
images_flatten_initial = images.reshape((2, 268203))
sum += len(filenames)
# for i in range(len(filenames)):
# true_label.append(filenames[i].split('-')[2][:-4])
adv_images, true_label = sess.run([x_adv, label],
feed_dict={x_input: images})
print('adv_images::::', type(adv_images[0][0][0][0]),
adv_images.shape)
Prediction = sess.run(pred_adv_v3,
feed_dict={x_adv_v3: adv_images})
print('true_label::::::::', true_label)
images_flatten_adv = adv_images.reshape((2, 268203))
l2_diatance_list = np.linalg.norm(
(images_flatten_initial - images_flatten_adv),
axis=1,
keepdims=True)
for n in range(len(filenames)):
l2_distance += l2_diatance_list[n]
for j in range(len(filenames)):
label_distance += abs(true_label[j] - Prediction[j])
if int(true_label[j]) == Prediction[
j]: # The test tag is always 1 more than the real tag, so I subtract 1 here
loss_num += 1
print('sucess_num:::::', loss_num)
print('Prediction:::::::', Prediction)
save_images(adv_images, filenames, FLAGS.output_dir)
print('sum::::', sum)
print('loss_num::::', loss_num)
rate_wrong = loss_num / sum
print('rate_wrong:::::::', rate_wrong)
print('l2_distance:::::::', l2_distance)
print('label_distance::::::', label_distance)
end = time.clock()
print('run time::::::::', end - start)
def train():
"""Train classification for a number of steps."""
with tf.Graph().as_default():
# Returns and create (if necessary) the global step tensor.
global_step = tf.train.get_or_create_global_step()
# Get images and labels for datasets.
# Force input pipeline to CPU:0 to avoid operations sometimes ending up on GPU and resulting in a slow down.
with tf.device('/cpu:0'):
# images:[batch size size channel], labels:[batch], filenames: nouseful
images, labels, _ = tf_input.input(is_random=True,
is_training=True)
labels = labels - 1
# Build a Graph that computes the logits predictions from the inference model.
# Attentation please, train mobilenet directly maybe very very difficult to convergence,
# you should do retrain(transfer learning)
with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
logits, _ = inception_v3.inception_v3(inputs=images,
is_training=True,
num_classes=NUM_CLASSES)
# Or for simply
# logits, _ = inception_v3.inception_v3(inputs=images, num_classes=NUM_CLASSES, is_training=True)
# Calculate loss.
loss = tf_inference.loss(logits, labels)
# Build a Graph that trains the model with one batch of examples and
# updates the model parameters.
train_op = tf_inference.train(loss, global_step)
class _LoggerHook(tf.train.SessionRunHook):
"""Logs loss and runtime."""
def begin(self):
self._step = -1
self._start_time = time.time()
def before_run(self, run_context):
self._step += 1
return tf.train.SessionRunArgs(loss) # Asks for loss value.
def after_run(self, run_context, run_values):
if self._step % FLAGS.log_frequency == 0:
current_time = time.time()
duration = current_time - self._start_time
self._start_time = current_time
loss_value = run_values.results
examples_per_sec = FLAGS.log_frequency * FLAGS.batch_size / duration
sec_per_batch = float(duration / FLAGS.log_frequency)
format_str = (
'%s: step %d, loss = %.2f (%.1f examples/sec; %.3f '
'sec/batch)')
print(format_str % (datetime.now(), self._step, loss_value,
examples_per_sec, sec_per_batch))
with tf.train.MonitoredTrainingSession(
checkpoint_dir=FLAGS.train_dir,
hooks=[
tf.train.StopAtStepHook(last_step=FLAGS.max_steps),
tf.train.NanTensorHook(loss),
_LoggerHook()
],
config=tf.ConfigProto(log_device_placement=FLAGS.
log_device_placement)) as mon_sess:
while not mon_sess.should_stop():
mon_sess.run(train_op)
def initialize(self):
if not hasattr(self.algo, '_kwargs'):
return
if 'ablation_type' in self.algo._kwargs:
self.ablation_type = self.algo._kwargs['ablation_type']
else:
self.ablation_type = "None"
#img = self.render('rgb_array')
if 'imsize' in self.algo._kwargs:
self.imsize = self.algo._kwargs['imsize']
self.name = self.algo._kwargs['name']
self.mode = self.algo._kwargs['mode']
if self.mode.startswith('inception'):
self.layer = self.algo._kwargs['layer']
batch_size = 25
idims = self.imsize
image = tf.placeholder(tf.uint8, (batch_size, ) + idims + (3, ),
name='image')
image_trans = tf.image.convert_image_dtype(image, dtype=tf.float32)
image_trans = tf.subtract(image_trans, 0.5)
image_trans = tf.multiply(image_trans, 2.0)
with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
model = inception_v3.inception_v3(image_trans,
num_classes=1001,
is_training=False,
dropout_keep_prob=1.0)
variables_to_restore = slim.get_variables_to_restore()
restorer = tf.train.Saver(variables_to_restore)
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
# sess.run(tf.global_variables_initializer())
restorer.restore(sess, "model/inception_v3.ckpt")
bird = scipy.misc.imread('model/bird.jpg')
bird = scipy.misc.imresize(bird, idims)
logits = sess.run(model[0], {image: [bird] * batch_size})
print(np.argsort(logits[0])[-20:])
self.sess = sess
self.image = image
self.model = model
if self.mode == 'inceptionsame':
with open(self.algo._kwargs['experttheano'], 'rb') as pfile:
expertpolicy = pickle.load(pfile)
allfeats = []
for nroll in range(20):
path = rollout(self.algo.env,
expertpolicy,
max_path_length=self.algo.max_path_length,
animated=False)
# import IPython
# IPython.embed()
imgs = [
img[0] for img in path['env_infos']['imgs']
if img is not None
]
feat = self.sess.run(self.model[1][self.layer],
{self.image: imgs})
allfeats.append(feat)
# import IPython
# IPython.embed()
self.means = np.mean(allfeats, axis=0)
self.std = np.std(allfeats, axis=0)
else:
data = np.load(self.algo._kwargs['meanfile'])
self.means = data[self.layer]
self.std = data[self.layer + 'std']
# print(efile, expertpolicy)
elif self.mode.startswith('ours'):
idim = self.imsize
self.batch_size = 25
tfinput = tf.placeholder(tf.uint8,
(3, self.batch_size) + idim + (3, ),
name='x')
image_trans = tf.image.convert_image_dtype(tfinput,
dtype=tf.float32)
image_trans = tf.subtract(image_trans, 0.5)
image_trans = tf.multiply(image_trans, 2.0)
self.image_trans = image_trans
if self.mode == 'oursinception':
with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
model = inception_v3.inception_v3(tf.reshape(
tensor=image_trans,
shape=(3 * self.batch_size, ) + idim + (3, )),
num_classes=1001,
is_training=False,
dropout_keep_prob=1.0)
autodc = arm_shaping.ContextAEInception2(
strides=[1, 2, 1, 2],
kernels=[3, 3, 3, 3],
filters=[1024, 1024, 512, 512])
featlayer = model[1]['Mixed_7c']
featshape = featlayer.get_shape().as_list()
featreshape = tf.reshape(featlayer,
(3, self.batch_size, featshape[1],
featshape[2], featshape[3]))
with tf.variable_scope("contextmodel") as scope:
autodc.build(featreshape)
self.image_trans = featreshape
else:
if self.name == 'real' or self.name == 'sweep':
autodc = arm_shaping.ContextAEReal()
elif self.name == 'push' or self.name == 'reach' or self.name == 'strike' or self.name == 'throw':
autodc = arm_shaping.ContextSkipNew()
autodc.build(image_trans)
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
learning_rate = tf.placeholder(tf.float32, shape=[])
# sess.run(tf.global_variables_initializer())
saver = tf.train.Saver()
saver.restore(sess, self.algo._kwargs['modelname'])
if self.mode == 'oursinception':
bird = scipy.misc.imread('model/bird.jpg')
bird = scipy.misc.imresize(bird, idim)
logits = sess.run(model[0],
{tfinput: [[bird] * self.batch_size] * 3})
print(np.argsort(logits[0])[-20:])
self.nvp = self.algo._kwargs['nvp']
self.sess = sess
self.image = tfinput
self.model = autodc
self.initialized = True
def run_training(path_db,
pid,
category,
task_id,
path_unknown,
pretrained_dir,
tensorflow_dir,
path_save,
num_epochs=1000,
batch_size=32,
finetune_last_layer=False,
data_augmentation=True,
mix_up=False,
network_model='inception-v3',
restore_all_parameters=False,
initial_learning_rate=0.0002,
learning_rate_decay_factor=0.7,
num_epochs_before_decay=2):
##### start parameters for creating TFRecord files #####
#validation_size = 0.1
validation_size = 0.0
num_shards = 2
random_seed = 0
##### end parameters for creating TFRecord files #####
dataset_dir = os.path.join(path_db, pid, category)
log_dir = path_save
tfrecord_filename = pid + '_' + category
if _dataset_exists(dataset_dir=dataset_dir,
_NUM_SHARDS=num_shards,
output_filename=tfrecord_filename):
print('Dataset files already exist. Overwrite them.')
photo_filenames, class_names = _get_filenames_and_classes(
dataset_dir, path_unknown)
# dictionary for class name and class ID
class_names_to_ids = dict(zip(class_names, range(len(class_names))))
# number of validation examples
num_validation = int(validation_size * len(photo_filenames))
# divide to training and validation data
random.seed(random_seed)
random.shuffle(photo_filenames)
training_filenames = photo_filenames[num_validation:]
validation_filenames = photo_filenames[:num_validation]
# find available GPU ID
gpu_id = gpu_utils.pick_gpu_lowest_memory()
# if log directory does not exist, create log directory and dataset
if not os.path.exists(log_dir):
os.makedirs(log_dir)
print('found lowest memory gpu id : ' + str(gpu_id))
_convert_dataset(gpu_id,
'train',
training_filenames,
class_names_to_ids,
dataset_dir=dataset_dir,
tfrecord_filename=tfrecord_filename,
_NUM_SHARDS=num_shards)
_convert_dataset(gpu_id,
'validation',
validation_filenames,
class_names_to_ids,
dataset_dir=dataset_dir,
tfrecord_filename=tfrecord_filename,
_NUM_SHARDS=num_shards)
labels_to_class_names = dict(zip(range(len(class_names)), class_names))
write_label_file(labels_to_class_names, dataset_dir)
print('finished creating dataset ' + tfrecord_filename)
# start training
output_label_filepath = os.path.join(dataset_dir, 'labels.txt')
if network_model!='inception-v4' and network_model!='inception-v3' and network_model!='resnet-v2-50' and network_model!='resnet-v2-152' and \
network_model!='vgg-16' and network_model!='mobilenet-v1' and network_model!='nasnet-large' and network_model!='nasnet-mobile':
print("invalid network model : " + network_model)
sys.exit()
# find pretrained model
if os.path.exists(os.path.join(log_dir, 'model.ckpt')):
checkpoint_file = os.path.join(log_dir, 'model.ckpt')
else:
if network_model == 'inception-v4':
checkpoint_file = os.path.join(
pretrained_dir, 'inception_resnet_v2_2016_08_30.ckpt')
elif network_model == 'inception-v3':
checkpoint_file = os.path.join(pretrained_dir, 'inception_v3.ckpt')
elif network_model == 'resnet-v2-50':
checkpoint_file = os.path.join(pretrained_dir, 'resnet_v2_50.ckpt')
elif network_model == 'resnet-v2-152':
checkpoint_file = os.path.join(pretrained_dir,
'resnet_v2_152.ckpt')
elif network_model == 'vgg-16':
checkpoint_file = os.path.join(pretrained_dir, 'vgg_16.ckpt')
elif network_model == 'mobilenet-v1':
checkpoint_file = os.path.join(pretrained_dir,
'mobilenet_v1_1.0_224.ckpt')
elif network_model == 'nasnet-large':
checkpoint_file = os.path.join(pretrained_dir,
'nasnet-a_large_04_10_2017',
'model.ckpt')
elif network_model == 'nasnet-mobile':
checkpoint_file = os.path.join(pretrained_dir,
'nasnet-a_mobile_04_10_2017',
'model.ckpt')
else:
print("invalid network model : " + network_model)
sys.exit()
# set image size
if network_model == 'inception-v4' or network_model == 'inception-v3' or network_model == 'resnet-v2-50' or network_model == 'resnet-v2-152':
image_size = 299
elif network_model == 'vgg-16' or network_model == 'mobilenet-v1' or network_model == 'nasnet-mobile':
image_size = 224
elif network_model == 'nasnet-large':
image_size = 331
else:
print("invalid network model : " + network_model)
sys.exit()
# create the file pattern of TFRecord files
file_pattern = tfrecord_filename + '_%s_*.tfrecord'
file_pattern_for_counting = tfrecord_filename
labels_to_name, label_list = load_labels(output_label_filepath)
num_classes = len(label_list)
# create a dataset discription
items_to_descriptions = {
'image':
'A 3-channel RGB coloured image that is either ' +
','.join(label_list),
'label':
'A label that is as such -- ' + ','.join([
str(key) + ':' + labels_to_name[key]
for key in labels_to_name.keys()
])
}
# start training
with tf.Graph().as_default() as graph:
tf.logging.set_verbosity(tf.logging.INFO)
# create dataset and load one batch
dataset = get_split('train', dataset_dir, file_pattern,
file_pattern_for_counting, labels_to_name,
num_classes, items_to_descriptions)
images, _, labels = load_batch(dataset,
batch_size=batch_size,
data_augmentation=data_augmentation,
mix_up=mix_up,
height=image_size,
width=image_size)
# number of steps to take before decaying the learning rate and batches per epoch
num_batches_per_epoch = int(dataset.num_samples / batch_size)
num_steps_per_epoch = num_batches_per_epoch # because one step is one batch processed
decay_steps = int(num_epochs_before_decay * num_steps_per_epoch)
# create model for inference
finetune_vars = []
if network_model == 'inception-v4':
with slim.arg_scope(inception_resnet_v2_arg_scope()):
logits, end_points = inception_resnet_v2(
images, num_classes=dataset.num_classes, is_training=True)
finetune_vars = [
'InceptionResnetV2/Logits', 'InceptionResnetV2/AuxLogits'
]
elif network_model == 'inception-v3':
with slim.arg_scope(inception_v3_arg_scope()):
logits, end_points = inception_v3(
images, num_classes=dataset.num_classes, is_training=True)
finetune_vars = ['InceptionV3/Logits', 'InceptionV3/AuxLogits']
elif network_model == 'resnet-v2-50':
with slim.arg_scope(resnet_arg_scope()):
logits, end_points = resnet_v2_50(
images, num_classes=dataset.num_classes, is_training=True)
finetune_vars = ['resnet_v2_50/logits']
elif network_model == 'resnet-v2-152':
with slim.arg_scope(resnet_arg_scope()):
logits, end_points = resnet_v2_152(
images, num_classes=dataset.num_classes, is_training=True)
finetune_vars = ['resnet_v2_152/logits']
elif network_model == 'vgg-16':
with slim.arg_scope(vgg_arg_scope()):
logits, _ = vgg_16(images,
num_classes=dataset.num_classes,
is_training=True)
finetune_vars = ['vgg_16/fc8']
elif network_model == 'mobilenet-v1':
with slim.arg_scope(mobilenet_v1_arg_scope()):
logits, end_points = mobilenet_v1(
images, num_classes=dataset.num_classes, is_training=True)
finetune_vars = ['MobilenetV1/Logits']
elif network_model == 'nasnet-large':
with slim.arg_scope(nasnet.nasnet_large_arg_scope()):
logits, end_points = nasnet.build_nasnet_large(
images, dataset.num_classes)
finetune_vars = [
'final_layer', 'aux_11',
'cell_stem_0/comb_iter_0/left/global_step'
]
elif network_model == 'nasnet-mobile':
with slim.arg_scope(nasnet.nasnet_mobile_arg_scope()):
logits, end_points = nasnet.build_nasnet_mobile(
images, dataset.num_classes)
finetune_vars = ['final_layer', 'aux_7']
else:
print("Invalid network model : " + network_model)
sys.exit()
# define the scopes that you want to exclude for restoration
exclude = []
if not restore_all_parameters:
exclude = finetune_vars
variables_to_restore = slim.get_variables_to_restore(exclude=exclude)
if mix_up:
labels.set_shape([batch_size, dataset.num_classes])
logits.set_shape([batch_size, dataset.num_classes])
loss = tf.losses.sigmoid_cross_entropy(labels, logits)
else:
# perform one-hot-encoding of the labels (Try one-hot-encoding within the load_batch function!)
one_hot_labels = slim.one_hot_encoding(labels, dataset.num_classes)
# performs the equivalent to tf.nn.sparse_softmax_cross_entropy_with_logits but enhanced with checks
loss = tf.losses.softmax_cross_entropy(
onehot_labels=one_hot_labels, logits=logits)
total_loss = tf.losses.get_total_loss(
) #obtain the regularization losses as well
# create the global step for monitoring the learning_rate and training.
global_step = tf.train.get_or_create_global_step()
# define your exponentially decaying learning rate
lr = tf.train.exponential_decay(learning_rate=initial_learning_rate,
global_step=global_step,
decay_steps=decay_steps,
decay_rate=learning_rate_decay_factor,
staircase=True)
# define optimizer
optimizer = tf.train.AdamOptimizer(learning_rate=lr)
# create train_op
if finetune_last_layer:
variables_to_train = get_variables_to_train_by_scopes(
finetune_vars)
print("finetune variables : " + str(variables_to_train))
train_op = slim.learning.create_train_op(
total_loss, optimizer, variables_to_train=variables_to_train)
else:
train_op = slim.learning.create_train_op(total_loss, optimizer)
# define prediction matrix
if network_model=='inception-v4' or network_model=='inception-v3' or network_model=='mobilenet-v1' or \
network_model=='nasnet-large' or network_model=='nasnet-mobile':
predictions = tf.argmax(end_points['Predictions'], 1)
probabilities = end_points['Predictions']
elif network_model == 'resnet-v2-50' or network_model == 'resnet-v2-152':
predictions = tf.argmax(end_points['predictions'], 1)
probabilities = end_points['predictions']
elif network_model == 'vgg-16':
predictions = tf.argmax(logits, 1)
probabilities = tf.nn.softmax(logits)
else:
print("Invalid network model : " + network_model)
sys.exit()
if mix_up:
argmax_labels = tf.argmax(labels, 1)
accuracy, accuracy_update = tf.contrib.metrics.streaming_accuracy(
predictions, argmax_labels)
else:
accuracy, accuracy_update = tf.contrib.metrics.streaming_accuracy(
predictions, labels)
metrics_op = tf.group(accuracy_update, probabilities)
# create summaries
tf.summary.scalar('losses/Total_Loss', total_loss)
tf.summary.scalar('accuracy', accuracy)
tf.summary.scalar('learning_rate', lr)
my_summary_op = tf.summary.merge_all()
# defube training step function that runs both the train_op, metrics_op and updates the global_step concurrently
def train_step(sess, train_op, global_step):
# check the time for each sess run
start_time = time.time()
total_loss, global_step_count, _ = sess.run(
[train_op, global_step, metrics_op])
time_elapsed = time.time() - start_time
# run the logging to print some results
logging.info('global step %s: loss: %.4f (%.2f sec/step)',
global_step_count, total_loss, time_elapsed)
return total_loss, int(global_step_count)
# create a saver function that actually restores the variables from a checkpoint file
saver = tf.train.Saver(variables_to_restore)
def restore_fn(sess):
return saver.restore(sess, checkpoint_file)
# define your supervisor for running a managed session
sv = tf.train.Supervisor(logdir=log_dir,
summary_op=None,
init_fn=restore_fn)
# run the managed session
start_train_time = time.time()
gpu_options = tf.ConfigProto(
gpu_options=tf.GPUOptions(visible_device_list=str(gpu_id),
per_process_gpu_memory_fraction=0.4))
with sv.prepare_or_wait_for_session(config=gpu_options) as sess:
for step in range(num_steps_per_epoch * num_epochs):
# check if training task is not canceled
if not controller.check_train_task_alive(
pid, category, task_id):
print('Training task is canceled.')
sv.stop()
return False, "", "", output_label_filepath, global_step_count
# at the start of every epoch, show the vital information:
if step % num_batches_per_epoch == 0:
logging.info('Epoch %s/%s',
step / num_batches_per_epoch + 1, num_epochs)
learning_rate_value, accuracy_value = sess.run(
[lr, accuracy])
logging.info('Current Learning Rate: %s',
learning_rate_value)
logging.info('Current Streaming Accuracy: %s',
accuracy_value)
# optionally, print your logits and predictions for a sanity check that things are going fine.
logits_value, probabilities_value, predictions_value, labels_value = sess.run(
[logits, probabilities, predictions, labels])
print('logits: \n', logits_value)
print('Probabilities: \n', probabilities_value)
print('predictions: \n', predictions_value)
print('Labels:\n:', labels_value)
# log the summaries every 10 step.
if step % 10 == 0:
loss, global_step_count = train_step(
sess, train_op, sv.global_step)
summaries = sess.run(my_summary_op)
sv.summary_computed(sess, summaries)
# if not, simply run the training step
else:
loss, global_step_count = train_step(
sess, train_op, sv.global_step)
# if specific time passes, save model for evaluation
time_elapsed_train = time.time() - start_train_time
print('training time : ' + str(time_elapsed_train))
# log the final training loss and accuracy
logging.info(
'Training Progress : %.2f %% ',
100.0 * step / float(num_steps_per_epoch * num_epochs))
logging.info('Final Loss: %s', loss)
logging.info('Global Step: %s', global_step_count)
logging.info('Final Accuracy: %s', sess.run(accuracy))
# after all the training has been done, save the log files and checkpoint model
logging.info('Finished training! Saving model to disk now.')
sv.saver.save(sess, sv.save_path, global_step=sv.global_step)
# save graph definition file
output_graph_filepath = os.path.join(log_dir, 'graph.pb')
export_graph_command_exec = "./network/export_slim_graph.py"
if not os.path.exists(export_graph_command_exec):
print("fatal error, cannot find command : " +
export_graph_command_exec)
sys.exit()
export_graph_command_env = os.environ.copy()
export_graph_command_env["CUDA_VISIBLE_DEVICES"] = ''
export_graph_command = []
export_graph_command.append(sys.executable)
export_graph_command.append(export_graph_command_exec)
export_graph_command.append(network_model)
export_graph_command.append(str(dataset.num_classes))
export_graph_command.append(output_graph_filepath)
print("start exec:" + " ".join(export_graph_command))
proc = subprocess.Popen(export_graph_command,
env=export_graph_command_env)
print("export graph process ID=" + str(proc.pid))
controller.upsert_train_child_process(task_id, proc.pid)
proc.communicate()
controller.delete_train_child_process(task_id, proc.pid)
print("finish exec:" + " ".join(export_graph_command))
if not controller.check_train_task_alive(pid, category, task_id):
print('Training task is canceled.')
sv.stop()
return False, "", "", output_label_filepath, global_step_count
# save frozon graph, optimized graph, and quantized graph from graph definition and checkpoint
latest_checkpoint_filepath = tf.train.latest_checkpoint(log_dir)
# you can check output node name by tensorflow/tools/graph_transforms::summarize_graph
# https://github.com/tensorflow/models/tree/master/research/slim#Export
output_node_names = ""
if network_model == 'inception-v4':
output_node_names = "InceptionResnetV2/Logits/Predictions"
elif network_model == 'inception-v3':
output_node_names = "InceptionV3/AuxLogits/SpatialSqueeze,InceptionV3/Predictions/Reshape_1"
elif network_model == 'resnet-v2-50':
output_node_names = "resnet_v2_50/predictions/Reshape_1"
elif network_model == 'resnet-v2-152':
output_node_names = "resnet_v2_152/predictions/Reshape_1"
elif network_model == 'vgg-16':
output_node_names = "vgg_16/fc8/squeezed"
elif network_model == 'mobilenet-v1':
output_node_names = "MobilenetV1/Predictions/Reshape_1"
elif network_model == 'nasnet-large' or network_model == 'nasnet-mobile':
output_node_names = "final_layer/predictions"
else:
print("Invalid network model : " + network_model)
sys.exit()
output_frozen_graph_filepath = os.path.join(
log_dir, 'frozen_graph.pb')
freeze_graph_command_exec = os.path.join(
tensorflow_dir,
"bazel-bin/tensorflow/python/tools/freeze_graph")
if not os.path.exists(freeze_graph_command_exec):
print("fatal error, cannot find command : " +
freeze_graph_command_exec)
sys.exit()
freeze_graph_command_env = os.environ.copy()
freeze_graph_command_env["CUDA_VISIBLE_DEVICES"] = ''
freeze_graph_command = []
freeze_graph_command.append(freeze_graph_command_exec)
freeze_graph_command.append("--input_graph=" +
output_graph_filepath)
freeze_graph_command.append("--input_checkpoint=" +
latest_checkpoint_filepath)
freeze_graph_command.append("--input_binary=true")
freeze_graph_command.append("--output_graph=" +
output_frozen_graph_filepath)
freeze_graph_command.append("--output_node_names=" +
output_node_names)
print("start exec:" + " ".join(freeze_graph_command))
proc = subprocess.Popen(freeze_graph_command,
env=freeze_graph_command_env)
print("freeze graph process ID=" + str(proc.pid))
controller.upsert_train_child_process(task_id, proc.pid)
proc.communicate()
controller.delete_train_child_process(task_id, proc.pid)
print("finish exec:" + " ".join(freeze_graph_command))
if not controller.check_train_task_alive(pid, category, task_id):
print('Training task is canceled.')
sv.stop()
return False, "", "", output_label_filepath, global_step_count
output_optimized_graph_filepath = os.path.join(
log_dir, 'optimized_graph.pb')
optimize_graph_command_exec = os.path.join(
tensorflow_dir,
"bazel-bin/tensorflow/python/tools/optimize_for_inference")
if not os.path.exists(optimize_graph_command_exec):
print("fatal error, cannot find command : " +
optimize_graph_command_exec)
sys.exit()
optimize_graph_command_env = os.environ.copy()
optimize_graph_command_env["CUDA_VISIBLE_DEVICES"] = ''
optimize_graph_command = []
optimize_graph_command.append(optimize_graph_command_exec)
optimize_graph_command.append("--input=" +
output_frozen_graph_filepath)
optimize_graph_command.append("--output=" +
output_optimized_graph_filepath)
optimize_graph_command.append("--input_names=input")
optimize_graph_command.append("--output_names=" +
output_node_names)
optimize_graph_command.append("--frozen_graph=true")
print("start exec:" + " ".join(optimize_graph_command))
proc = subprocess.Popen(optimize_graph_command,
env=optimize_graph_command_env)
print("optimize graph process ID=" + str(proc.pid))
controller.upsert_train_child_process(task_id, proc.pid)
proc.communicate()
controller.delete_train_child_process(task_id, proc.pid)
print("finish exec:" + " ".join(optimize_graph_command))
if not controller.check_train_task_alive(pid, category, task_id):
print('Training task is canceled.')
sv.stop()
return False, "", "", output_label_filepath, global_step_count
output_quantized_graph_filepath = os.path.join(
log_dir, 'quantized_graph.pb')
quantize_graph_command_exec = os.path.join(
tensorflow_dir,
"bazel-bin/tensorflow/tools/quantization/quantize_graph")
if not os.path.exists(quantize_graph_command_exec):
print("fatal error, cannot find command : " +
quantize_graph_command_exec)
sys.exit()
quantize_graph_command_env = os.environ.copy()
quantize_graph_command_env["CUDA_VISIBLE_DEVICES"] = ''
quantize_graph_command = []
quantize_graph_command.append(quantize_graph_command_exec)
quantize_graph_command.append("--input=" +
output_optimized_graph_filepath)
quantize_graph_command.append("--output=" +
output_quantized_graph_filepath)
quantize_graph_command.append("--input_node_names=input")
quantize_graph_command.append("--output_node_names=" +
output_node_names)
quantize_graph_command.append("--mode=eightbit")
print("start exec:" + " ".join(quantize_graph_command))
proc = subprocess.Popen(quantize_graph_command,
env=quantize_graph_command_env)
print("quantize graph process ID=" + str(proc.pid))
controller.upsert_train_child_process(task_id, proc.pid)
proc.communicate()
controller.delete_train_child_process(task_id, proc.pid)
print("finish exec:" + " ".join(quantize_graph_command))
if not controller.check_train_task_alive(pid, category, task_id):
print('Training task is canceled.')
sv.stop()
return False, "", "", output_label_filepath, global_step_count
return True, output_optimized_graph_filepath, output_quantized_graph_filepath, output_label_filepath, global_step_count
def graph(x, mask, y, i, x_max, x_min, grad):
eps = 2.0 * FLAGS.max_epsilon / 255.0
alpha = eps / 10
momentum = FLAGS.momentum
num_classes = 1001
with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
logits_v3, end_points_v3 = inception_v3.inception_v3(
input_diversity(x), num_classes=num_classes, is_training=False)
logits_v3_rotated, _ = inception_v3.inception_v3(
rotate(x), num_classes=num_classes, is_training=False, reuse=True)
with slim.arg_scope(inception_v4.inception_v4_arg_scope()):
logits_v4, end_points_v4 = inception_v4.inception_v4(
input_diversity(x), num_classes=num_classes, is_training=False)
logits_v4_rotated, _ = inception_v4.inception_v4(
rotate(x), num_classes=num_classes, is_training=False, reuse=True)
with slim.arg_scope(inception_resnet_v2.inception_resnet_v2_arg_scope()):
logits_res_v2, end_points_res_v2 = inception_resnet_v2.inception_resnet_v2(
input_diversity(x),
num_classes=num_classes,
is_training=False,
reuse=True)
logits_res_v2_rotated, _ = inception_resnet_v2.inception_resnet_v2(
rotate(x), num_classes=num_classes, is_training=False, reuse=True)
with slim.arg_scope(resnet_v2.resnet_arg_scope()):
logits_resnet, end_points_resnet = resnet_v2.resnet_v2_152(
input_diversity(x), num_classes=num_classes, is_training=False)
logits_resnet_rotated, _ = resnet_v2.resnet_v2_152(
rotate(x), num_classes=num_classes, is_training=False, reuse=True)
with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
logits_ens3_adv_v3, end_points_ens3_adv_v3 = inception_v3.inception_v3(
input_diversity(x),
num_classes=num_classes,
is_training=False,
scope='Ens3AdvInceptionV3')
logits_ens3_adv_v3_rotated, _ = inception_v3.inception_v3(
rotate(x),
num_classes=num_classes,
is_training=False,
scope='Ens3AdvInceptionV3',
reuse=True)
with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
logits_ens4_adv_v3, end_points_ens4_adv_v3 = inception_v3.inception_v3(
input_diversity(x),
num_classes=num_classes,
is_training=False,
scope='Ens4AdvInceptionV3')
logits_ens4_adv_v3_rotated, _ = inception_v3.inception_v3(
rotate(x),
num_classes=num_classes,
is_training=False,
scope='Ens4AdvInceptionV3',
reuse=True)
# with slim.arg_scope(inception_resnet_v2.inception_resnet_v2_arg_scope()):
# logits_ens_advres_v2, end_points_ens_advres_v2 = inception_resnet_v2.inception_resnet_v2(
# input_diversity(x), num_classes=num_classes, is_training=False, scope='EnsAdvInceptionResnetV2')
# logits_ens_advres_v2_rotated, _ = inception_resnet_v2.inception_resnet_v2(
# rotate(x), num_classes=num_classes, is_training=False, scope='EnsAdvInceptionResnetV2', reuse=True)
logits = (logits_v3 + logits_v4 + logits_res_v2 + logits_resnet +
logits_v3_rotated + logits_v4_rotated + logits_res_v2_rotated +
logits_resnet_rotated + logits_ens3_adv_v3 + logits_ens4_adv_v3 +
logits_ens3_adv_v3_rotated + logits_ens4_adv_v3_rotated) / 12
auxlogits = (
end_points_v3['AuxLogits'] + end_points_v4['AuxLogits'] +
end_points_res_v2['AuxLogits'] + end_points_ens3_adv_v3['AuxLogits'] +
end_points_ens4_adv_v3['AuxLogits']
# + end_points_ens_advres_v2['AuxLogits']
) / 5
cross_entropy = tf.losses.softmax_cross_entropy(y,
logits,
label_smoothing=0.0,
weights=1.0)
cross_entropy += tf.losses.softmax_cross_entropy(y,
auxlogits,
label_smoothing=0.0,
weights=0.4)
noise = tf.gradients(cross_entropy, x)[0]
noise = tf.nn.depthwise_conv2d(noise,
stack_kernel,
strides=[1, 1, 1, 1],
padding='SAME')
noise = noise / tf.reduce_mean(tf.abs(noise), [1, 2, 3], keep_dims=True)
noise = momentum * grad + noise
noise = noise * mask[:, :, :, np.newaxis]
x = x + alpha * tf.sign(noise)
x = tf.clip_by_value(x, x_min, x_max)
i = tf.add(i, 1)
return x, mask, y, i, x_max, x_min, noise
def graph_large(x, target_class_input, i, x_max, x_min, grad):
eps = 2.0 * FLAGS.max_epsilon / 255.0
alpha = eps / 12
momentum = FLAGS.momentum
num_classes = 1001
with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
logits_v3, end_points_v3 = inception_v3.inception_v3(
x, num_classes=num_classes, is_training=False)
with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
logits_adv_v3, end_points_adv_v3 = inception_v3.inception_v3(
x,
num_classes=num_classes,
is_training=False,
scope='AdvInceptionV3')
with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
logits_ens3_adv_v3, end_points_ens3_adv_v3 = inception_v3.inception_v3(
x,
num_classes=num_classes,
is_training=False,
scope='Ens3AdvInceptionV3')
with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
logits_ens4_adv_v3, end_points_ens4_adv_v3 = inception_v3.inception_v3(
x,
num_classes=num_classes,
is_training=False,
scope='Ens4AdvInceptionV3')
with slim.arg_scope(inception_resnet_v2.inception_resnet_v2_arg_scope()):
logits_ensadv_res_v2, end_points_ensadv_res_v2 = inception_resnet_v2.inception_resnet_v2(
x,
num_classes=num_classes,
is_training=False,
scope='EnsAdvInceptionResnetV2')
one_hot_target_class = tf.one_hot(target_class_input, num_classes)
logits = (4 * logits_v3 + logits_adv_v3 + logits_ens3_adv_v3 +
logits_ens4_adv_v3 + 4 * logits_ensadv_res_v2) / 11
auxlogits = (4 * end_points_v3['AuxLogits'] +
end_points_adv_v3['AuxLogits'] +
end_points_ens3_adv_v3['AuxLogits'] +
end_points_ens4_adv_v3['AuxLogits'] +
4 * end_points_ensadv_res_v2['AuxLogits']) / 11
cross_entropy = tf.losses.softmax_cross_entropy(one_hot_target_class,
logits,
label_smoothing=0.0,
weights=1.0)
cross_entropy += tf.losses.softmax_cross_entropy(one_hot_target_class,
auxlogits,
label_smoothing=0.0,
weights=0.4)
noise = tf.gradients(cross_entropy, x)[0]
noise = noise / tf.reshape(
tf.contrib.keras.backend.std(tf.reshape(noise, [FLAGS.batch_size, -1]),
axis=1), [FLAGS.batch_size, 1, 1, 1])
noise = momentum * grad + noise
noise = noise / tf.reshape(
tf.contrib.keras.backend.std(tf.reshape(noise, [FLAGS.batch_size, -1]),
axis=1), [FLAGS.batch_size, 1, 1, 1])
x = x - alpha * tf.clip_by_value(tf.round(noise), -2, 2)
x = tf.clip_by_value(x, x_min, x_max)
i = tf.add(i, 1)
return x, target_class_input, i, x_max, x_min, noise
def graph(x, y, i, x_max, x_min, grad):
eps = 2.0 * FLAGS.max_epsilon / 255.0
num_iter = FLAGS.num_iter
alpha = eps / num_iter
momentum = FLAGS.momentum
num_classes = 1001
x_nes = x
with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
logits_v3, end_points_v3 = inception_v3.inception_v3(
x_nes,
num_classes=num_classes,
is_training=False,
reuse=tf.AUTO_REUSE)
pred = tf.argmax(end_points_v3['Predictions'], 1)
first_round = tf.cast(tf.equal(i, 0), tf.int64)
y = first_round * pred + (1 - first_round) * y
one_hot = tf.one_hot(y, num_classes)
cross_entropy = tf.losses.softmax_cross_entropy(one_hot, logits_v3)
noise = tf.gradients(cross_entropy, x)[0]
x_nes_2 = 1 / 2 * x_nes
with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
logits_v3_2, end_points_v3 = inception_v3.inception_v3(
x_nes_2,
num_classes=num_classes,
is_training=False,
reuse=tf.AUTO_REUSE)
cross_entropy_2 = tf.losses.softmax_cross_entropy(one_hot, logits_v3_2)
noise += tf.gradients(cross_entropy_2, x)[0]
x_nes_4 = 1 / 4 * x_nes
with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
logits_v3_4, end_points_v3 = inception_v3.inception_v3(
x_nes_4,
num_classes=num_classes,
is_training=False,
reuse=tf.AUTO_REUSE)
cross_entropy_4 = tf.losses.softmax_cross_entropy(one_hot, logits_v3_4)
noise += tf.gradients(cross_entropy_4, x)[0]
x_nes_8 = 1 / 8 * x_nes
with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
logits_v3_8, end_points_v3 = inception_v3.inception_v3(
x_nes_8,
num_classes=num_classes,
is_training=False,
reuse=tf.AUTO_REUSE)
cross_entropy_8 = tf.losses.softmax_cross_entropy(one_hot, logits_v3_8)
noise += tf.gradients(cross_entropy_8, x)[0]
x_nes_16 = 1 / 16 * x_nes
with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
logits_v3_16, end_points_v3 = inception_v3.inception_v3(
x_nes_16,
num_classes=num_classes,
is_training=False,
reuse=tf.AUTO_REUSE)
cross_entropy_16 = tf.losses.softmax_cross_entropy(one_hot, logits_v3_16)
noise += tf.gradients(cross_entropy_16, x)[0]
noise = noise / tf.reduce_mean(tf.abs(noise), [1, 2, 3], keep_dims=True)
noise = momentum * grad + noise
x = x + alpha * tf.sign(noise)
x = tf.clip_by_value(x, x_min, x_max)
i = tf.add(i, 1)
return x, y, i, x_max, x_min, noise
def main(_):
# Images for inception classifier are normalized to be in [-1, 1] interval,
# eps is a difference between pixels so it should be in [0, 2] interval.
# Renormalizing epsilon from [0, 255] to [0, 2].
eps = 2.0 * FLAGS.max_epsilon / 255.0
num_classes = 1001
batch_shape = [FLAGS.batch_size, FLAGS.image_height, FLAGS.image_width, 3]
tf.logging.set_verbosity(tf.logging.INFO)
with tf.Graph().as_default():
tf.set_random_seed(305)
# Prepare graph
x_input = tf.placeholder(tf.float32, shape = batch_shape)
adv_img = tf.placeholder(tf.float32, shape = batch_shape)
y = tf.placeholder(tf.int32, shape = batch_shape[0])
t_y = tf.placeholder(tf.int32, shape = batch_shape[0])
x_max = tf.clip_by_value(x_input + eps, -1.0, 1.0)
x_min = tf.clip_by_value(x_input - eps, -1.0, 1.0)
with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
logits_v3, end_points_v3 = inception_v3.inception_v3(
adv_img, num_classes=num_classes, is_training=False)
pre_v3 = tf.argmax(logits_v3, 1)
with slim.arg_scope(inception_v4.inception_v4_arg_scope()):
logits_v4, end_points_v4 = inception_v4.inception_v4(
adv_img, num_classes=num_classes, is_training=False)
pre_v4 = tf.argmax(logits_v4, 1)
with slim.arg_scope(resnet_v2.resnet_arg_scope()):
logits_resnet_152, end_points_resnet = resnet_v2.resnet_v2_152(
adv_img, num_classes=num_classes, is_training=False)
pre_resnet_152 = tf.argmax(logits_resnet_152, 1)
with slim.arg_scope(resnet_v2.resnet_arg_scope()):
logits_resnet_101, end_points_resnet_101 = resnet_v2.resnet_v2_101(
adv_img, num_classes=num_classes, is_training=False)
pre_resnet_101 = tf.argmax(logits_resnet_101, 1)
with slim.arg_scope(resnet_v2.resnet_arg_scope()):
logits_resnet_50, end_points_resnet_50 = resnet_v2.resnet_v2_50(
adv_img, num_classes=num_classes, is_training=False)
pre_resnet_50 = tf.argmax(logits_resnet_50, 1)
with slim.arg_scope(inception_resnet_v2.inception_resnet_v2_arg_scope()):
logits_Incres, end_points_IR = inception_resnet_v2.inception_resnet_v2(
adv_img, num_classes=num_classes, is_training=False)
pre_Inc_res = tf.argmax(logits_Incres, 1)
with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
logits_ens3_adv_v3, end_points_ens3_adv_v3 = inception_v3.inception_v3(
adv_img, num_classes=num_classes, is_training=False, scope='Ens3AdvInceptionV3')
pre_ens3_adv_v3 = tf.argmax(logits_ens3_adv_v3, 1)
with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
logits_ens4_adv_v3, end_points_ens4_adv_v3 = inception_v3.inception_v3(
adv_img, num_classes=num_classes, is_training=False, scope='Ens4AdvInceptionV3')
pre_ens4_adv_v3 = tf.argmax(logits_ens4_adv_v3, 1)
with slim.arg_scope(inception_resnet_v2.inception_resnet_v2_arg_scope()):
logits_ensadv_res_v2, end_points_ensadv_res_v2 = inception_resnet_v2.inception_resnet_v2(
adv_img, num_classes=num_classes, is_training=False, scope='EnsAdvInceptionResnetV2')
pre_ensadv_res_v2 = tf.argmax(logits_ensadv_res_v2, 1)
pre_ensemble_logit = tf.argmax((logits_resnet_152 + logits_v4 + logits_v3 + + logits_resnet_101 + logits_resnet_50 +\
logits_Incres + logits_ens3_adv_v3 + logits_ens4_adv_v3), 1)
mean_pert = 0.0
sum_v3, sum_v4, sum_res152, sum_res101, sum_res50, sum_Incres, sum_ensemble = 0, 0, 0, 0, 0, 0, 0
sum_ens3_adv_v3, sum_ens4_adv_v3, sum_ensadv_res_v2 = 0, 0, 0
i = tf.constant(0)
grad = tf.zeros(shape=batch_shape)
amplification = tf.zeros(shape=batch_shape)
_, x_adv, _, _, _, _, _, _, _ = tf.while_loop(stop, graph, [x_input, adv_img, y, t_y, i, x_max, x_min, grad, amplification])
# Run computation
s1 = tf.train.Saver(slim.get_model_variables(scope='InceptionV3'))
s3 = tf.train.Saver(slim.get_model_variables(scope='Ens3AdvInceptionV3'))
s4 = tf.train.Saver(slim.get_model_variables(scope='Ens4AdvInceptionV3'))
s5 = tf.train.Saver(slim.get_model_variables(scope='InceptionV4'))
s6 = tf.train.Saver(slim.get_model_variables(scope='InceptionResnetV2'))
s7 = tf.train.Saver(slim.get_model_variables(scope='EnsAdvInceptionResnetV2'))
s8 = tf.train.Saver(slim.get_model_variables(scope='resnet_v2_152'))
s9 = tf.train.Saver(slim.get_model_variables(scope='resnet_v2_101'))
s10 = tf.train.Saver(slim.get_model_variables(scope='resnet_v2_50'))
with tf.Session() as sess:
s1.restore(sess, model_checkpoint_map['inception_v3'])
s3.restore(sess, model_checkpoint_map['ens3_adv_inception_v3'])
s4.restore(sess, model_checkpoint_map['ens4_adv_inception_v3'])
s5.restore(sess, model_checkpoint_map['inception_v4'])
s6.restore(sess, model_checkpoint_map['inception_resnet_v2'])
s7.restore(sess, model_checkpoint_map['ens_adv_inception_resnet_v2'])
s8.restore(sess, model_checkpoint_map['resnet_v2_152'])
s9.restore(sess, model_checkpoint_map['resnet_v2_101'])
s10.restore(sess, model_checkpoint_map['resnet_v2_50'])
import pandas as pd
dev = pd.read_csv(FLAGS.input_csv)
for idx in tqdm(range(0, 1000 // FLAGS.batch_size)):
images, filenames, True_label, Target_label = load_images(FLAGS.input_dir, dev, idx * FLAGS.batch_size, batch_shape)
my_adv_images = sess.run(x_adv, feed_dict={x_input: images, adv_img: images, y: True_label, t_y: Target_label}).astype(np.float32)
mean_pert += abs(my_adv_images - images).mean()
pre_v3_, pre_v4_, pre_resnet152_, pre_resnet101_, pre_resnet50_, pre_Inc_res_,\
pre_ens3_adv_v3_, pre_ens4_adv_v3_, pre_ensadv_res_v2_ ,pre_ensemble_ \
= sess.run([pre_v3, pre_v4, pre_resnet_152, pre_resnet_101, pre_resnet_50, pre_Inc_res,
pre_ens3_adv_v3, pre_ens4_adv_v3, pre_ensadv_res_v2,
pre_ensemble_logit], feed_dict = {adv_img: my_adv_images})
sum_v3 += (pre_v3_ == Target_label).sum()
sum_v4 += (pre_v4_ == Target_label).sum()
sum_res152 += (pre_resnet152_ == Target_label).sum()
sum_res101 += (pre_resnet101_ == Target_label).sum()
sum_res50 += (pre_resnet50_ == Target_label).sum()
sum_Incres += (pre_Inc_res_ == Target_label).sum()
sum_ens3_adv_v3 += (pre_ens3_adv_v3_ == Target_label).sum()
sum_ens4_adv_v3 += (pre_ens4_adv_v3_ == Target_label).sum()
sum_ensadv_res_v2 += (pre_ensadv_res_v2_ == Target_label).sum()
sum_ensemble += (pre_ensemble_ == Target_label).sum()
save_images(my_adv_images, filenames, FLAGS.output_dir)
print('mean noise = ', (mean_pert / (1000.0 / FLAGS.batch_size)) * 255.0)
print('sum_v3 = {}'.format(sum_v3))
print('sum_v4 = {}'.format(sum_v4))
print('sum_res2 = {}'.format(sum_res152))
print('sum_res1 = {}'.format(sum_res101))
print('sum_res1 = {}'.format(sum_res50))
print('sum_Incres_v2 = {}'.format(sum_Incres))
print('sum_ens3_adv_v3 = {}'.format(sum_ens3_adv_v3))
print('sum_ens4_adv_v3 = {}'.format(sum_ens4_adv_v3))
print('sum_ensadv_Incres_v2 = {}'.format(sum_ensadv_res_v2))
print('sum_ensmeble = {}'.format(sum_ensemble))
def main(_):
batch_shape = [FLAGS.batch_size, FLAGS.image_height, FLAGS.image_width, 3]
num_classes = 1001
itr = 30
tf.logging.set_verbosity(tf.logging.INFO)
with tf.Graph().as_default():
# Prepare graph
x_input = tf.placeholder(tf.float32, shape=batch_shape)
img_resize_tensor = tf.placeholder(tf.int32, [2])
x_input_resize = tf.image.resize_images(
x_input,
img_resize_tensor,
method=tf.image.ResizeMethod.NEAREST_NEIGHBOR)
shape_tensor = tf.placeholder(tf.int32, [3])
padded_input = padding_layer_iyswim(x_input_resize, shape_tensor)
# 330 is the last value to keep 8*8 output, 362 is the last value to keep 9*9 output, stride = 32
padded_input.set_shape(
(FLAGS.batch_size, FLAGS.image_resize, FLAGS.image_resize, 3))
with slim.arg_scope(
inception_resnet_v2.inception_resnet_v2_arg_scope()):
logits_ensadv_res_v2, end_points_ensadv_res_v2 = inception_resnet_v2.inception_resnet_v2(
padded_input,
num_classes=num_classes,
is_training=False,
create_aux_logits=True)
with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
logits_adv_v3, end_points_v3 = inception_v3.inception_v3(
padded_input,
num_classes=num_classes,
is_training=False,
scope='AdvInceptionV3')
with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
logits_ens3_adv_v3, end_points_ens3_adv_v3 = inception_v3.inception_v3(
padded_input,
num_classes=num_classes,
is_training=False,
scope='Ens3AdvInceptionV3')
with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
logits_ens3_adv_v3, end_points_ens4_adv_v3 = inception_v3.inception_v3(
padded_input,
num_classes=num_classes,
is_training=False,
scope='Ens4AdvInceptionV3')
with slim.arg_scope(resnet_v2.resnet_arg_scope()):
logits_resnet, end_points_resnet = resnet_v2.resnet_v2_101(
padded_input, num_classes=num_classes, is_training=False)
logits = (logits_ensadv_res_v2 + logits_adv_v3 + logits_ens3_adv_v3 +
logits_ens3_adv_v3 + logits_resnet) / 5
Aux = (end_points_ensadv_res_v2['AuxLogits'] +
end_points_v3['AuxLogits'] + end_points_ens3_adv_v3['AuxLogits']
+ end_points_ens4_adv_v3['AuxLogits']) * 0.1
predicted_labels = tf.argmax((logits + Aux), 1)
#predicted_labels = tf.argmax(end_points['Predictions'], 1)
# Run computation
#saver = tf.train.Saver(slim.get_model_variables())
#session_creator = tf.train.ChiefSessionCreator(
# scaffold=tf.train.Scaffold(saver=saver),
# checkpoint_filename_with_path=[FLAGS.checkpoint_path_ens3_adv_inception_v3,FLAGS.checkpoint_path_ens4_adv_inception_v3]
# master=FLAGS.master)
#with tf.train.MonitoredSession(session_creator=session_creator) as sess:
s1 = tf.train.Saver(slim.get_model_variables(scope='AdvInceptionV3'))
s2 = tf.train.Saver(
slim.get_model_variables(scope='Ens3AdvInceptionV3'))
s3 = tf.train.Saver(
slim.get_model_variables(scope='Ens4AdvInceptionV3'))
s4 = tf.train.Saver(
slim.get_model_variables(scope='EnsAdvInceptionResnetV2'))
s5 = tf.train.Saver(slim.get_model_variables(scope='resnet_v2'))
with tf.Session() as sess:
s1.restore(sess, FLAGS.checkpoint_path_adv_inception_v3)
s2.restore(sess, FLAGS.checkpoint_path_ens3_adv_inception_v3)
s3.restore(sess, FLAGS.checkpoint_path_ens4_adv_inception_v3)
s4.restore(sess, FLAGS.checkpoint_path_ens_adv_inception_resnet_v2)
s5.restore(sess, FLAGS.checkpoint_path_resnet)
with tf.gfile.Open(FLAGS.output_file, 'w') as out_file:
for filenames, images in load_images(FLAGS.input_dir,
batch_shape):
final_preds = np.zeros(
[FLAGS.batch_size, num_classes, itr])
for j in range(itr):
if np.random.randint(0, 2, size=1) == 1:
images = images[:, :, ::-1, :]
resize_shape_ = np.random.randint(310, 331)
final_preds[..., j] = sess.run(
[predicted_labels],
feed_dict={
x_input:
images,
img_resize_tensor: [resize_shape_] * 2,
shape_tensor:
np.array([
random.randint(
0, FLAGS.image_resize - resize_shape_),
random.randint(
0, FLAGS.image_resize - resize_shape_),
FLAGS.image_resize
])
})
final_probs = np.sum(final_preds, axis=-1)
labels = np.argmax(final_probs, 1)
for filename, label in zip(filenames, labels):
out_file.write('{0},{1}\n'.format(filename, label))
def graph(x, adv, y, t_y, i, x_max, x_min, grad, amplification):
target_one_hot = tf.one_hot(t_y, 1001)
true_one_hot = tf.one_hot(y, 1001)
eps = 2.0 * FLAGS.max_epsilon / 255.0
num_iter = FLAGS.num_iter
alpha = eps / num_iter
alpha_beta = alpha * 10.0
gamma = alpha_beta * 0.8
momentum = FLAGS.momentum
num_classes = 1001
# input_diversity(FLAGS, adv): for DI-FGSM
with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
logits_v3, end_points_v3 = inception_v3.inception_v3(
input_diversity(FLAGS, adv), num_classes=num_classes, is_training=False, reuse=True)
auxlogit_v3 = end_points_v3['AuxLogits']
with slim.arg_scope(inception_v4.inception_v4_arg_scope()):
logits_v4, end_points_v4 = inception_v4.inception_v4(
input_diversity(FLAGS, adv), num_classes=num_classes, is_training=False, reuse=True)
auxlogit_v4 = end_points_v4['AuxLogits']
with slim.arg_scope(resnet_v2.resnet_arg_scope()):
logits_resnet_152, end_points_resnet = resnet_v2.resnet_v2_152(
input_diversity(FLAGS, adv), num_classes=num_classes, is_training=False, reuse=True)
with slim.arg_scope(resnet_v2.resnet_arg_scope()):
logits_resnet_101, end_points_resnet_101 = resnet_v2.resnet_v2_101(
input_diversity(FLAGS, adv), num_classes=num_classes, is_training=False, reuse=True)
with slim.arg_scope(resnet_v2.resnet_arg_scope()):
logits_resnet_50, end_points_resnet_50 = resnet_v2.resnet_v2_50(
input_diversity(FLAGS, adv), num_classes=num_classes, is_training=False, reuse=True)
with slim.arg_scope(inception_resnet_v2.inception_resnet_v2_arg_scope()):
logits_Incres, end_points_IR = inception_resnet_v2.inception_resnet_v2(
input_diversity(FLAGS, adv), num_classes=num_classes, is_training=False, reuse=True)
auxlogit_IR = end_points_IR['AuxLogits']
with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
logits_ens3_adv_v3, end_points_ens3_adv_v3 = inception_v3.inception_v3(
input_diversity(FLAGS, adv), num_classes=num_classes, is_training=False, scope='Ens3AdvInceptionV3',reuse=True)
auxlogit_ens3 = end_points_ens3_adv_v3['AuxLogits']
with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
logits_ens4_adv_v3, end_points_ens4_adv_v3 = inception_v3.inception_v3(
input_diversity(FLAGS, adv), num_classes=num_classes, is_training=False, scope='Ens4AdvInceptionV3', reuse=True)
auxlogit_ens4 = end_points_ens4_adv_v3['AuxLogits']
# with slim.arg_scope(inception_resnet_v2.inception_resnet_v2_arg_scope()):
# logits_ensadv_res_v2, end_points_ensadv_res_v2 = inception_resnet_v2.inception_resnet_v2(
# input_diversity(FLAGS, adv), num_classes=num_classes, is_training=False, scope='EnsAdvInceptionResnetV2', reuse=True)
# auxlogit_resv2 = end_points_ensadv_res_v2['AuxLogits']
# FLAGS.temperature: for PI-FGSM++
logits = (logits_v4 + logits_resnet_152 + logits_v3 + logits_resnet_101 + logits_resnet_50 + \
logits_Incres + logits_ens3_adv_v3 + logits_ens4_adv_v3) / 8.0 / FLAGS.temperature
auxlogits = (auxlogit_v4 + auxlogit_v3 + auxlogit_IR + auxlogit_ens3 + auxlogit_ens4) / 5.0 / FLAGS.temperature
target_cross_entropy = tf.losses.softmax_cross_entropy(target_one_hot,
logits,
label_smoothing=0.0,
weights=1.0)
target_cross_entropy += tf.losses.softmax_cross_entropy(target_one_hot,
auxlogits,
label_smoothing=0.0,
weights=1.0)
noise = tf.gradients(target_cross_entropy, adv)[0]
noise = tf.nn.depthwise_conv2d(noise, T_kern, strides=[1, 1, 1, 1], padding='SAME')
# for MI-FGSM
# noise = noise / tf.reduce_mean(tf.abs(noise), [1, 2, 3], keep_dims=True)
# noise = momentum * grad + noise
amplification += alpha_beta * n_staircase_sign(noise, num_of_K)
cut_noise = tf.clip_by_value(abs(amplification) - eps, 0.0, 10000.0) * tf.sign(amplification)
projection = gamma * n_staircase_sign(project_noise(cut_noise, P_kern, kern_size), num_of_K)
adv = adv - alpha_beta * n_staircase_sign(noise, num_of_K) - projection
# adv = adv - alpha * n_staircase_sign(noise, num_of_K)
adv = tf.clip_by_value(adv, x_min, x_max)
i = tf.add(i, 1)
return x, adv, y, t_y, i, x_max, x_min, noise, amplification,
def Train(input_dir, output_dir):
# some parameter
batch_shape = [batch_size, FLAGS.img_size, FLAGS.img_size, 3]
with tf.Graph().as_default():
# Prepare graph
train_img = tf.placeholder(
tf.float32, shape=[None, FLAGS.img_size, FLAGS.img_size, 3])
train_label = tf.placeholder(tf.float32, shape=[None, 2])
with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
logits, end_points = inception_v3.inception_v3(train_img,
num_classes=2,
is_training=True,
scope=model_type)
predict = tf.argmax(end_points['Predictions'], 1)
logits = end_points['Logits']
cost = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(labels=train_label,
logits=logits))
learning_rate = tf.placeholder(tf.float32, name='learning_rate')
optimizer = tf.train.GradientDescentOptimizer(
learning_rate=learning_rate)
train_step = optimizer.minimize(cost)
accuracy = tf.reduce_mean(
tf.cast(tf.equal(predict, tf.argmax(train_label, 1)), tf.float32))
# Run computation
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
if FineTune:
exclusion = [
'%s/Logits' % model_type,
'%s/AuxLogits' % model_type
] #去除class类里的描述
except_logitis = slim.get_variables_to_restore(
exclude=exclusion)
#print(except_logitis)
init_fn = slim.assign_from_checkpoint_fn(
FLAGS.model, except_logitis, ignore_missing_vars=True)
init_fn(sess)
saver = tf.train.Saver(
slim.get_model_variables(scope=model_type))
else:
saver = tf.train.Saver(
slim.get_model_variables(scope=model_type))
saver.restore(sess, FLAGS.model)
for time in range(epoch_num):
train_acc = []
count = 0
train_loss = []
val_acc = []
val_loss = []
epoch_learning_rate = init_learning_rate
for raw_images, true_labels in load_images_with_true_label(
input_dir, train=True):
labels = one_hot(np.array(true_labels), 2)
img = np.array((raw_images / 255.0) * 2.0 - 1.0)
img = np.reshape(
img, [batch_size, FLAGS.img_size, FLAGS.img_size, 3])
train_feed_dict = {
train_img: img,
train_label: labels,
learning_rate: epoch_learning_rate
}
sess.run(train_step, feed_dict=train_feed_dict)
batch_loss = cost.eval(feed_dict=train_feed_dict)
batch_acc = accuracy.eval(feed_dict=train_feed_dict)
train_acc.append(batch_acc)
train_loss.append(batch_loss)
count += 1
if count % 500 == 0:
print('acc: ', np.mean(np.array(train_acc)), ' loss: ',
np.mean(np.array(train_loss)))
break
saver.save(sess=sess,
save_path='./train_model/%s_%s.ckpt' %
(model_type, time))
for raw_images, true_labels in load_images_with_true_label(
output_dir, train=False):
labels = one_hot(np.array(true_labels), 2)
img = np.array((raw_images / 255.0) * 2.0 - 1.0)
img = np.reshape(
img, [batch_size, FLAGS.img_size, FLAGS.img_size, 3])
train_feed_dict = {
train_img: img,
train_label: labels,
learning_rate: epoch_learning_rate
}
batch_acc = sess.run([accuracy], feed_dict=train_feed_dict)
val_acc.append(batch_acc)
count += 1
print('val_acc: ', np.mean(np.array(val_acc)))
def main(_):
# Images for inception classifier are normalized to be in [-1, 1] interval,
# eps is a difference between pixels so it should be in [0, 2] interval.
# Renormalizing epsilon from [0, 255] to [0, 2].
tf.logging.set_verbosity(tf.logging.INFO)
full_start = timer()
batch_shape = [FLAGS.batch_size, FLAGS.image_height, FLAGS.image_width, 3]
all_images_taget_class, all_images_true_label = load_target_class(
FLAGS.input_dir)
if not os.path.exists(FLAGS.output_dir):
os.mkdir(FLAGS.output_dir)
with tf.Graph().as_default():
# Prepare graph
x_input = tf.placeholder(tf.float32, shape=batch_shape)
target_class_input = tf.placeholder(tf.int32, shape=[FLAGS.batch_size])
momentum = FLAGS.momentum
eps = 2.0 * FLAGS.max_epsilon / 255.0
alpha = 0.2
num_classes = 1000
num_classes_a = 1001
# image = x_input
image = input_diversity(x_input)
# image = batch_dct2d(image)
"""
224 input
"""
processed_imgs_res_v1_50 = preprocess_for_model(image, 'resnet_v1_50')
with slim.arg_scope(resnet_v1.resnet_arg_scope()):
logits_res_v1_50, end_points_res_v1_50 = resnet_v1.resnet_v1_50(
processed_imgs_res_v1_50,
num_classes=num_classes,
is_training=False,
reuse=tf.AUTO_REUSE)
processed_imgs_res_v1_101 = preprocess_for_model(
image, 'resnet_v1_101')
with slim.arg_scope(resnet_v1.resnet_arg_scope()):
logits_res_v1_101, end_points_res_v1_101 = resnet_v1.resnet_v1_101(
processed_imgs_res_v1_101,
num_classes=num_classes,
is_training=False,
reuse=tf.AUTO_REUSE)
processed_res_v1 = preprocess_for_model(image, 'resnet_v1_152')
with slim.arg_scope(resnet_v1.resnet_arg_scope()):
logits_res_v1_152, end_points_res_v1 = resnet_v1.resnet_v1_152(
processed_res_v1,
num_classes=num_classes,
is_training=False,
scope='resnet_v1_152',
reuse=tf.AUTO_REUSE)
processed_imgs_vgg_16 = preprocess_for_model(image, 'vgg_16')
with slim.arg_scope(vgg.vgg_arg_scope()):
logits_vgg_16, end_points_vgg_16 = vgg.vgg_16(
processed_imgs_vgg_16,
num_classes=num_classes,
is_training=False,
scope='vgg_16')
processed_imgs_vgg_19 = preprocess_for_model(image, 'vgg_19')
with slim.arg_scope(vgg.vgg_arg_scope()):
logits_vgg_19, end_points_vgg_19 = vgg.vgg_19(
processed_imgs_vgg_19,
num_classes=num_classes,
is_training=False,
scope='vgg_19')
logits_clean_a = (logits_res_v1_50 + logits_res_v1_101 +
logits_res_v1_152 + logits_vgg_16 +
logits_vgg_19) / 5.0
processed_imgs_inception_v1 = preprocess_for_model(
image, 'inception_v1')
with slim.arg_scope(inception_v1.inception_v1_arg_scope()):
logits_inception_v1, end_points_inception_v1 = inception_v1.inception_v1(
processed_imgs_inception_v1,
num_classes=num_classes_a,
is_training=False,
reuse=tf.AUTO_REUSE)
"""
299 input
"""
x_div = preprocess_for_model(image, 'inception_v3')
with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
logits_inc_v3, end_points_inc_v3 = inception_v3.inception_v3(
x_div,
num_classes=num_classes_a,
is_training=False,
scope='InceptionV3')
with slim.arg_scope(inception_v4.inception_v4_arg_scope()):
logits_inc_v4, end_points_inc_v4 = inception_v4.inception_v4(
x_div,
num_classes=num_classes_a,
is_training=False,
scope='InceptionV4')
with slim.arg_scope(
inception_resnet_v2.inception_resnet_v2_arg_scope()):
logits_inc_res_v2, end_points_inc_res_v2 = inception_resnet_v2.inception_resnet_v2(
x_div,
num_classes=num_classes_a,
is_training=False,
scope='InceptionResnetV2')
logits_clean_b = (logits_inc_v3 + logits_inc_v4 + logits_inc_res_v2 +
logits_inception_v1) / 4.0
"""
add adv model
"""
with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
logits_adv_v3, end_points_adv_v3 = inception_v3.inception_v3(
x_div,
num_classes=num_classes_a,
is_training=False,
scope='AdvInceptionV3')
with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
logits_ens3_adv_v3, end_points_ens3_adv_v3 = inception_v3.inception_v3(
x_div,
num_classes=num_classes_a,
is_training=False,
scope='Ens3AdvInceptionV3')
with slim.arg_scope(inception_v3.inception_v3_arg_scope()):
logits_ens4_adv_v3, end_points_ens4_adv_v3 = inception_v3.inception_v3(
x_div,
num_classes=num_classes_a,
is_training=False,
scope='Ens4AdvInceptionV3')
with slim.arg_scope(
inception_resnet_v2.inception_resnet_v2_arg_scope()):
logits_ensadv_res_v2, end_points_ensadv_res_v2 = inception_resnet_v2.inception_resnet_v2(
x_div,
num_classes=num_classes_a,
is_training=False,
scope='EnsAdvInceptionResnetV2')
with slim.arg_scope(
inception_resnet_v2.inception_resnet_v2_arg_scope()):
logits_adv_res_v2, end_points_adv_res_v2 = inception_resnet_v2.inception_resnet_v2(
x_div,
num_classes=num_classes_a,
is_training=False,
scope='AdvInceptionResnetV2')
logits_ens_a = (logits_adv_v3 + logits_ens3_adv_v3 + logits_ens4_adv_v3
+ logits_ensadv_res_v2 + logits_adv_res_v2) / 5.0
logits_ens_aux = (end_points_adv_v3['AuxLogits'] +
end_points_ens3_adv_v3['AuxLogits'] +
end_points_ens4_adv_v3['AuxLogits'] +
end_points_adv_res_v2['AuxLogits'] +
end_points_ensadv_res_v2['AuxLogits']) / 5.0
label_test = tf.argmax(logits_adv_v3, axis=1)
"""
ensemble model loss
"""
clean_logits = (logits_clean_a + logits_clean_b[:, 1:1001]) / 2.0
adv_logits = logits_ens_a[:, 1:1001] + logits_ens_aux[:, 1:1001]
logits = (clean_logits + adv_logits) / 2.0
ens_labels = tf.argmax(logits, axis=1)
one_hot = tf.one_hot(target_class_input, num_classes)
loss_adv_v3 = tf.losses.softmax_cross_entropy(one_hot,
logits_adv_v3[:, 1:1001],
label_smoothing=0.0,
weights=1.0)
loss_ens3_adv_v3 = tf.losses.softmax_cross_entropy(
one_hot,
logits_ens3_adv_v3[:, 1:1001],
label_smoothing=0.0,
weights=1.0)
loss_ens4_adv_v3 = tf.losses.softmax_cross_entropy(
one_hot,
logits_ens4_adv_v3[:, 1:1001],
label_smoothing=0.0,
weights=1.0)
loss_ensadv_res_v2 = tf.losses.softmax_cross_entropy(
one_hot,
logits_ensadv_res_v2[:, 1:1001],
label_smoothing=0.0,
weights=1.0)
loss_adv_res_v2 = tf.losses.softmax_cross_entropy(
one_hot,
logits_adv_res_v2[:, 1:1001],
label_smoothing=0.0,
weights=1.0)
loss_res_v1_101 = tf.losses.softmax_cross_entropy(one_hot,
logits_res_v1_101,
label_smoothing=0.0,
weights=1.0)
loss_res_v1_50 = tf.losses.softmax_cross_entropy(one_hot,
logits_res_v1_50,
label_smoothing=0.0,
weights=1.0)
loss_vgg_16 = tf.losses.softmax_cross_entropy(one_hot,
logits_vgg_16,
label_smoothing=0.0,
weights=1.0)
loss_res_v1_152 = tf.losses.softmax_cross_entropy(one_hot,
logits_res_v1_152,
label_smoothing=0.0,
weights=1.0)
total_loss = tf.losses.softmax_cross_entropy(one_hot,
logits,
label_smoothing=0.0,
weights=1.0)
noise = tf.gradients(total_loss, x_input)[0]
kernel = gkern(15, FLAGS.sig).astype(np.float32)
stack_kernel = np.stack([kernel, kernel, kernel]).swapaxes(2, 0)
stack_kernel = np.expand_dims(stack_kernel, 3)
noise = tf.nn.depthwise_conv2d(noise,
stack_kernel,
strides=[1, 1, 1, 1],
padding='SAME')
# [batch, out_height, out_width, in_channels * channel_multiplier]
noise = noise / tf.reshape(
tf.contrib.keras.backend.std(tf.reshape(noise,
[FLAGS.batch_size, -1]),
axis=1), [FLAGS.batch_size, 1, 1, 1])
# noise = momentum * grad + noise
noise = noise / tf.reshape(
tf.contrib.keras.backend.std(tf.reshape(noise,
[FLAGS.batch_size, -1]),
axis=1), [FLAGS.batch_size, 1, 1, 1])
s1 = tf.train.Saver(slim.get_model_variables(scope='InceptionV1'))
s2 = tf.train.Saver(slim.get_model_variables(scope='InceptionV3'))
s3 = tf.train.Saver(slim.get_model_variables(scope='InceptionV4'))
s4 = tf.train.Saver(slim.get_model_variables(scope='resnet_v1_50'))
s5 = tf.train.Saver(slim.get_model_variables(scope='resnet_v1_101'))
s6 = tf.train.Saver(slim.get_model_variables(scope='resnet_v1_152'))
s7 = tf.train.Saver(slim.get_model_variables(scope='vgg_16'))
s8 = tf.train.Saver(slim.get_model_variables(scope='vgg_19'))
s9 = tf.train.Saver(
slim.get_model_variables(scope='InceptionResnetV2'))
s10 = tf.train.Saver(
slim.get_model_variables(scope='AdvInceptionResnetV2'))
s11 = tf.train.Saver(
slim.get_model_variables(scope='Ens3AdvInceptionV3'))
s12 = tf.train.Saver(
slim.get_model_variables(scope='Ens4AdvInceptionV3'))
s13 = tf.train.Saver(
slim.get_model_variables(scope='EnsAdvInceptionResnetV2'))
s14 = tf.train.Saver(slim.get_model_variables(scope='AdvInceptionV3'))
print('Created Graph')
with tf.Session() as sess:
s1.restore(sess, FLAGS.checkpoint_path_inception_v1)
s2.restore(sess, FLAGS.checkpoint_path_inception_v3)
s3.restore(sess, FLAGS.checkpoint_path_inception_v4)
s4.restore(sess, FLAGS.checkpoint_path_resnet_v1_50)
s5.restore(sess, FLAGS.checkpoint_path_resnet_v1_101)
s6.restore(sess, FLAGS.checkpoint_path_resnet_v1_152)
s7.restore(sess, FLAGS.checkpoint_path_vgg_16)
s8.restore(sess, FLAGS.checkpoint_path_vgg_19)
s9.restore(sess, FLAGS.checkpoint_path_inception_resnet_v2)
s10.restore(sess, FLAGS.checkpoint_path_adv_inception_resnet_v2)
s11.restore(sess, FLAGS.checkpoint_path_ens3_adv_inception_v3)
s12.restore(sess, FLAGS.checkpoint_path_ens4_adv_inception_v3)
s13.restore(sess,
FLAGS.checkpoint_path_ens_adv_inception_resnet_v2)
s14.restore(sess, FLAGS.checkpoint_path_adv_inception_v3)
print('Initialized Models')
processed = 0.0
defense, tgt, untgt, final = 0.0, 0.0, 0.0, 0.0
idx = 0
for filenames, images in load_images(FLAGS.input_dir, batch_shape):
target_class_for_batch = (
[all_images_taget_class[n[:-4]] for n in filenames] + [0] *
(FLAGS.batch_size - len(filenames)))
true_label_for_batch = (
[all_images_true_label[n[:-4]] for n in filenames] + [0] *
(FLAGS.batch_size - len(filenames)))
x_max = np.clip(images + eps, -1.0, 1.0)
x_min = np.clip(images - eps, -1.0, 1.0)
adv_img = np.copy(images)
for i in range(FLAGS.iterations):
# loss_set = sess.run([loss_adv_v3,loss_ens3_adv_v3,loss_ens4_adv_v3,loss_ensadv_res_v2,
# loss_adv_res_v2,loss_res_v1_101,loss_res_v1_50,loss_vgg_16,loss_res_v1_152],
# feed_dict={x_input: batch_NLM(adv_img),
# target_class_input: target_class_for_batch})
# print ("loss:",loss_set)
# label_ens_model = sess.run([a,b,c,d],feed_dict={x_input: adv_img,target_class_input: target_class_for_batch})
# print ("label_ens_model:",label_ens_model)
# print (target_class_for_batch,true_label_for_batch)
adv_img = batch_NLM(adv_img) if i % 5 == 0 else adv_img
ens_loss, pred, grad, pred_adv_v3 = sess.run(
[total_loss, ens_labels, noise, label_test],
feed_dict={
x_input: adv_img,
target_class_input: target_class_for_batch
})
adv_img = adv_img - alpha * np.clip(np.round(grad), -2, 2)
adv_img = np.clip(adv_img, x_min, x_max)
print("{} \t total_loss {}".format(i, ens_loss))
print('prediction :', pred)
print('target_label :', target_class_for_batch)
print('true_label :', true_label_for_batch)
# print ("{} \t total_loss {} predction {} \t target class {} \t true label {} \t ".format(i,ens_loss,pred,target_class_for_batch,true_label_for_batch))
# print ("model predction {} \t target class {} \t true label {} \t ".format(pred,target_class_for_batch,true_label_for_batch))
print(
"final prediction {} \t target class {} \t true label {} \t "
.format(pred, target_class_for_batch,
true_label_for_batch))
processed += FLAGS.batch_size
tgt += sum(
np.equal(np.array(pred), np.array(target_class_for_batch)))
defense += sum(
np.equal(np.array(pred), np.array(true_label_for_batch)))
untgt = processed - tgt - defense
print("processed {} \t acc {} {} \t tgt {} {} \t untgt {} {} ".
format(processed, defense, defense / processed, tgt,
tgt / processed, untgt, untgt / processed))
full_end = timer()
print("DONE: Processed {} images in {} sec".format(
processed, full_end - full_start))
save_images(adv_img, filenames, FLAGS.output_dir)
print("DONE: Processed {} images in {} sec".format(
processed, full_end - full_start))
