def testEndPointsV2(self):
"""Test the end points of a tiny v2 bottleneck network."""
blocks = [
resnet_v2.resnet_v2_block(
'block1', base_depth=1, num_units=2, stride=2),
resnet_v2.resnet_v2_block(
'block2', base_depth=2, num_units=2, stride=1),
]
inputs = create_test_input(2, 32, 16, 3)
with slim.arg_scope(resnet_utils.resnet_arg_scope()):
_, end_points = self._resnet_plain(inputs, blocks, scope='tiny')
expected = [
'tiny/block1/unit_1/bottleneck_v2/shortcut',
'tiny/block1/unit_1/bottleneck_v2/conv1',
'tiny/block1/unit_1/bottleneck_v2/conv2',
'tiny/block1/unit_1/bottleneck_v2/conv3',
'tiny/block1/unit_2/bottleneck_v2/conv1',
'tiny/block1/unit_2/bottleneck_v2/conv2',
'tiny/block1/unit_2/bottleneck_v2/conv3',
'tiny/block2/unit_1/bottleneck_v2/shortcut',
'tiny/block2/unit_1/bottleneck_v2/conv1',
'tiny/block2/unit_1/bottleneck_v2/conv2',
'tiny/block2/unit_1/bottleneck_v2/conv3',
'tiny/block2/unit_2/bottleneck_v2/conv1',
'tiny/block2/unit_2/bottleneck_v2/conv2',
'tiny/block2/unit_2/bottleneck_v2/conv3']
self.assertItemsEqual(expected, end_points)
def testAtrousFullyConvolutionalValues(self):
"""Verify dense feature extraction with atrous convolution."""
nominal_stride = 32
for output_stride in [4, 8, 16, 32, None]:
with slim.arg_scope(resnet_utils.resnet_arg_scope()):
with tf.Graph().as_default():
with self.test_session() as sess:
tf.set_random_seed(0)
inputs = create_test_input(2, 81, 81, 3)
# Dense feature extraction followed by subsampling.
output, _ = self._resnet_small(inputs, None,
is_training=False,
global_pool=False,
output_stride=output_stride)
if output_stride is None:
factor = 1
else:
factor = nominal_stride // output_stride
output = resnet_utils.subsample(output, factor)
# Make the two networks use the same weights.
tf.get_variable_scope().reuse_variables()
# Feature extraction at the nominal network rate.
expected, _ = self._resnet_small(inputs, None,
is_training=False,
global_pool=False)
sess.run(tf.global_variables_initializer())
self.assertAllClose(output.eval(), expected.eval(),
atol=1e-4, rtol=1e-4)
def _extract_box_classifier_features(self, proposal_feature_maps, scope):
"""Extracts second stage box classifier features.
Args:
proposal_feature_maps: A 4-D float tensor with shape
[batch_size * self.max_num_proposals, crop_height, crop_width, depth]
representing the feature map cropped to each proposal.
scope: A scope name (unused).
Returns:
proposal_classifier_features: A 4-D float tensor with shape
[batch_size * self.max_num_proposals, height, width, depth]
representing box classifier features for each proposal.
"""
with tf.variable_scope(self._architecture, reuse=self._reuse_weights):
with slim.arg_scope(
resnet_utils.resnet_arg_scope(
batch_norm_epsilon=1e-5,
batch_norm_scale=True,
weight_decay=self._weight_decay)):
with slim.arg_scope([slim.batch_norm],
is_training=self._train_batch_norm):
blocks = [
resnet_utils.Block('block4', resnet_v1.bottleneck, [{
'depth': 2048,
'depth_bottleneck': 512,
'stride': 1
}] * 3)
]
proposal_classifier_features = resnet_utils.stack_blocks_dense(
proposal_feature_maps, blocks)
return proposal_classifier_features
def testClassificationEndPoints(self):
global_pool = True
num_classes = 10
inputs = create_test_input(2, 224, 224, 3)
with slim.arg_scope(resnet_utils.resnet_arg_scope()):
logits, end_points = self._resnet_small(inputs, num_classes,
global_pool=global_pool,
scope='resnet')
self.assertTrue(logits.op.name.startswith('resnet/logits'))
self.assertListEqual(logits.get_shape().as_list(), [2, 1, 1, num_classes])
self.assertTrue('predictions' in end_points)
self.assertListEqual(end_points['predictions'].get_shape().as_list(),
[2, 1, 1, num_classes])
def testFullyConvolutionalUnknownHeightWidth(self):
batch = 2
height, width = 65, 65
global_pool = False
inputs = create_test_input(batch, None, None, 3)
with slim.arg_scope(resnet_utils.resnet_arg_scope()):
output, _ = self._resnet_small(inputs, None, global_pool=global_pool)
self.assertListEqual(output.get_shape().as_list(),
[batch, None, None, 32])
images = create_test_input(batch, height, width, 3)
with self.test_session() as sess:
sess.run(tf.global_variables_initializer())
output = sess.run(output, {inputs: images.eval()})
self.assertEqual(output.shape, (batch, 3, 3, 32))
def _extract_proposal_features(self, preprocessed_inputs, scope):
"""Extracts first stage RPN features.
Args:
preprocessed_inputs: A [batch, height, width, channels] float32 tensor
representing a batch of images.
scope: A scope name.
Returns:
rpn_feature_map: A tensor with shape [batch, height, width, depth]
activations: A dictionary mapping feature extractor tensor names to
tensors
Raises:
InvalidArgumentError: If the spatial size of `preprocessed_inputs`
(height or width) is less than 33.
ValueError: If the created network is missing the required activation.
"""
if len(preprocessed_inputs.get_shape().as_list()) != 4:
raise ValueError('`preprocessed_inputs` must be 4 dimensional, got a '
'tensor of shape %s' % preprocessed_inputs.get_shape())
shape_assert = tf.Assert(
tf.logical_and(
tf.greater_equal(tf.shape(preprocessed_inputs)[1], 33),
tf.greater_equal(tf.shape(preprocessed_inputs)[2], 33)),
['image size must at least be 33 in both height and width.'])
with tf.control_dependencies([shape_assert]):
# Disables batchnorm for fine-tuning with smaller batch sizes.
# TODO(chensun): Figure out if it is needed when image
# batch size is bigger.
with slim.arg_scope(
resnet_utils.resnet_arg_scope(
batch_norm_epsilon=1e-5,
batch_norm_scale=True,
weight_decay=self._weight_decay)):
with tf.variable_scope(
self._architecture, reuse=self._reuse_weights) as var_scope:
_, activations = self._resnet_model(
preprocessed_inputs,
num_classes=None,
is_training=self._train_batch_norm,
global_pool=False,
output_stride=self._first_stage_features_stride,
spatial_squeeze=False,
scope=var_scope)
handle = scope + '/%s/block3' % self._architecture
return activations[handle], activations
def testClassificationShapes(self):
global_pool = True
num_classes = 10
inputs = create_test_input(2, 224, 224, 3)
with slim.arg_scope(resnet_utils.resnet_arg_scope()):
_, end_points = self._resnet_small(inputs, num_classes,
global_pool=global_pool,
scope='resnet')
endpoint_to_shape = {
'resnet/block1': [2, 28, 28, 4],
'resnet/block2': [2, 14, 14, 8],
'resnet/block3': [2, 7, 7, 16],
'resnet/block4': [2, 7, 7, 32]}
for endpoint in endpoint_to_shape:
shape = endpoint_to_shape[endpoint]
self.assertListEqual(end_points[endpoint].get_shape().as_list(), shape)
def testFullyConvolutionalEndpointShapes(self):
global_pool = False
num_classes = 10
inputs = create_test_input(2, 321, 321, 3)
with slim.arg_scope(resnet_utils.resnet_arg_scope()):
_, end_points = self._resnet_small(inputs, num_classes,
global_pool=global_pool,
scope='resnet')
endpoint_to_shape = {
'resnet/block1': [2, 41, 41, 4],
'resnet/block2': [2, 21, 21, 8],
'resnet/block3': [2, 11, 11, 16],
'resnet/block4': [2, 11, 11, 32]}
for endpoint in endpoint_to_shape:
shape = endpoint_to_shape[endpoint]
self.assertListEqual(end_points[endpoint].get_shape().as_list(), shape)
def _atrousValues(self, bottleneck):
"""Verify the values of dense feature extraction by atrous convolution.
Make sure that dense feature extraction by stack_blocks_dense() followed by
subsampling gives identical results to feature extraction at the nominal
network output stride using the simple self._stack_blocks_nondense() above.
Args:
bottleneck: The bottleneck function.
"""
blocks = [
resnet_utils.Block('block1', bottleneck, [(4, 1, 1), (4, 1, 2)]),
resnet_utils.Block('block2', bottleneck, [(8, 2, 1), (8, 2, 2)]),
resnet_utils.Block('block3', bottleneck, [(16, 4, 1), (16, 4, 2)]),
resnet_utils.Block('block4', bottleneck, [(32, 8, 1), (32, 8, 1)])
]
nominal_stride = 8
# Test both odd and even input dimensions.
height = 30
width = 31
with slim.arg_scope(resnet_utils.resnet_arg_scope()):
with slim.arg_scope([slim.batch_norm], is_training=False):
for output_stride in [1, 2, 4, 8, None]:
with tf.Graph().as_default():
with self.test_session() as sess:
tf.set_random_seed(0)
inputs = create_test_input(1, height, width, 3)
# Dense feature extraction followed by subsampling.
output = resnet_utils.stack_blocks_dense(inputs,
blocks,
output_stride)
if output_stride is None:
factor = 1
else:
factor = nominal_stride // output_stride
output = resnet_utils.subsample(output, factor)
# Make the two networks use the same weights.
tf.get_variable_scope().reuse_variables()
# Feature extraction at the nominal network rate.
expected = self._stack_blocks_nondense(inputs, blocks)
sess.run(tf.global_variables_initializer())
output, expected = sess.run([output, expected])
self.assertAllClose(output, expected, atol=1e-4, rtol=1e-4)
def testUnknownBatchSize(self):
batch = 2
height, width = 65, 65
global_pool = True
num_classes = 10
inputs = create_test_input(None, height, width, 3)
with slim.arg_scope(resnet_utils.resnet_arg_scope()):
logits, _ = self._resnet_small(inputs, num_classes,
global_pool=global_pool,
scope='resnet')
self.assertTrue(logits.op.name.startswith('resnet/logits'))
self.assertListEqual(logits.get_shape().as_list(),
[None, 1, 1, num_classes])
images = create_test_input(batch, height, width, 3)
with self.test_session() as sess:
sess.run(tf.global_variables_initializer())
output = sess.run(logits, {inputs: images.eval()})
self.assertEqual(output.shape, (batch, 1, 1, num_classes))
def testRootlessFullyConvolutionalEndpointShapes(self):
global_pool = False
num_classes = 10
inputs = create_test_input(2, 128, 128, 3)
with slim.arg_scope(resnet_utils.resnet_arg_scope()):
_, end_points = self._resnet_small(inputs, num_classes,
global_pool=global_pool,
include_root_block=False,
spatial_squeeze=False,
scope='resnet')
endpoint_to_shape = {
'resnet/block1': [2, 64, 64, 4],
'resnet/block2': [2, 32, 32, 8],
'resnet/block3': [2, 16, 16, 16],
'resnet/block4': [2, 16, 16, 32]}
for endpoint in endpoint_to_shape:
shape = endpoint_to_shape[endpoint]
self.assertListEqual(end_points[endpoint].get_shape().as_list(), shape)
def testEndpointNames(self):
# Like ResnetUtilsTest.testEndPointsV1(), but for the public API.
global_pool = True
num_classes = 10
inputs = create_test_input(2, 224, 224, 3)
with slim.arg_scope(resnet_utils.resnet_arg_scope()):
_, end_points = self._resnet_small(inputs, num_classes,
global_pool=global_pool,
scope='resnet')
expected = ['resnet/conv1']
for block in range(1, 5):
for unit in range(1, 4 if block < 4 else 3):
for conv in range(1, 4):
expected.append('resnet/block%d/unit_%d/bottleneck_v1/conv%d' %
(block, unit, conv))
expected.append('resnet/block%d/unit_%d/bottleneck_v1' % (block, unit))
expected.append('resnet/block%d/unit_1/bottleneck_v1/shortcut' % block)
expected.append('resnet/block%d' % block)
expected.extend(['global_pool', 'resnet/logits', 'resnet/spatial_squeeze',
'predictions'])
self.assertItemsEqual(end_points.keys(), expected)
def testEndPointsV2(self):
"""Test the end points of a tiny v2 bottleneck network."""
bottleneck = resnet_v2.bottleneck
blocks = [resnet_utils.Block('block1', bottleneck, [(4, 1, 1), (4, 1, 2)]),
resnet_utils.Block('block2', bottleneck, [(8, 2, 1), (8, 2, 1)])]
inputs = create_test_input(2, 32, 16, 3)
with slim.arg_scope(resnet_utils.resnet_arg_scope()):
_, end_points = self._resnet_plain(inputs, blocks, scope='tiny')
expected = [
'tiny/block1/unit_1/bottleneck_v2/shortcut',
'tiny/block1/unit_1/bottleneck_v2/conv1',
'tiny/block1/unit_1/bottleneck_v2/conv2',
'tiny/block1/unit_1/bottleneck_v2/conv3',
'tiny/block1/unit_2/bottleneck_v2/conv1',
'tiny/block1/unit_2/bottleneck_v2/conv2',
'tiny/block1/unit_2/bottleneck_v2/conv3',
'tiny/block2/unit_1/bottleneck_v2/shortcut',
'tiny/block2/unit_1/bottleneck_v2/conv1',
'tiny/block2/unit_1/bottleneck_v2/conv2',
'tiny/block2/unit_1/bottleneck_v2/conv3',
'tiny/block2/unit_2/bottleneck_v2/conv1',
'tiny/block2/unit_2/bottleneck_v2/conv2',
'tiny/block2/unit_2/bottleneck_v2/conv3']
self.assertItemsEqual(expected, end_points)
def testStridingLastUnitVsSubsampleBlockEnd(self):
"""Compares subsampling at the block's last unit or block's end.
Makes sure that the final output is the same when we use a stride at the
last unit of a block vs. we subsample activations at the end of a block.
"""
block = resnet_v1.resnet_v1_block
blocks = [
block('block1', base_depth=1, num_units=2, stride=2),
block('block2', base_depth=2, num_units=2, stride=2),
block('block3', base_depth=4, num_units=2, stride=2),
block('block4', base_depth=8, num_units=2, stride=1),
]
# Test both odd and even input dimensions.
height = 30
width = 31
with slim.arg_scope(resnet_utils.resnet_arg_scope()):
with slim.arg_scope([slim.batch_norm], is_training=False):
for output_stride in [1, 2, 4, 8, None]:
with tf.Graph().as_default():
with self.test_session() as sess:
tf.set_random_seed(0)
inputs = create_test_input(1, height, width, 3)
# Subsampling at the last unit of the block.
output = resnet_utils.stack_blocks_dense(
inputs, blocks, output_stride,
store_non_strided_activations=False,
outputs_collections='output')
output_end_points = slim.utils.convert_collection_to_dict(
'output')
# Make the two networks use the same weights.
tf.get_variable_scope().reuse_variables()
# Subsample activations at the end of the blocks.
expected = resnet_utils.stack_blocks_dense(
inputs, blocks, output_stride,
store_non_strided_activations=True,
outputs_collections='expected')
expected_end_points = slim.utils.convert_collection_to_dict(
'expected')
sess.run(tf.global_variables_initializer())
# Make sure that the final output is the same.
output, expected = sess.run([output, expected])
self.assertAllClose(output, expected, atol=1e-4, rtol=1e-4)
# Make sure that intermediate block activations in
# output_end_points are subsampled versions of the corresponding
# ones in expected_end_points.
for i, block in enumerate(blocks[:-1:]):
output = output_end_points[block.scope]
expected = expected_end_points[block.scope]
atrous_activated = (output_stride is not None and
2 ** i >= output_stride)
if not atrous_activated:
expected = resnet_utils.subsample(expected, 2)
output, expected = sess.run([output, expected])
self.assertAllClose(output, expected, atol=1e-4, rtol=1e-4)
"""
if len(preprocessed_inputs.get_shape().as_list()) != 4: //输入维度必须是4维度的
raise ValueError('`preprocessed_inputs` must be 4 dimensional, got a '
'tensor of shape %s' % preprocessed_inputs.get_shape())
shape_assert = tf.Assert( //如果预处理的输入图片的宽高小于33,会抛异常
tf.logical_and(
tf.greater_equal(tf.shape(preprocessed_inputs)[1], 33),
tf.greater_equal(tf.shape(preprocessed_inputs)[2], 33)),
['image size must at least be 33 in both height and width.'])
with tf.control_dependencies([shape_assert]):
# Disables batchnorm for fine-tuning with smaller batch sizes.
# TODO: Figure out if it is needed when image batch size is bigger.
with slim.arg_scope(
resnet_utils.resnet_arg_scope(
batch_norm_epsilon=1e-5,
batch_norm_scale=True,
weight_decay=self._weight_decay)):
with tf.variable_scope(
self._architecture, reuse=self._reuse_weights) as var_scope:
_, activations = self._resnet_model(
preprocessed_inputs,
num_classes=None,
is_training=False,
global_pool=False,
output_stride=self._first_stage_features_stride,
spatial_squeeze=False,
scope=var_scope)
handle = scope + '/%s/block3' % self._architecture
return activations[handle]
def train():
eps = 2.0 * float(FLAGS.max_epsilon) / 256.0
tf.logging.set_verbosity(tf.logging.INFO)
with tf.Graph().as_default():
# Design architecture
# input
x_data = tf.placeholder(tf.float32,
[None, FLAGS.img_height, FLAGS.img_width, 3],
name="x_data")
y_label = tf.placeholder(tf.float32, [None, FLAGS.num_classes],
name="y_label")
# generator
x_generated, g_params = build_generator(x_data, FLAGS)
x_generated = x_generated * eps
x_generated = x_data + x_generated
# discriminator(inception v3)
with slim.arg_scope(inception.inception_v3_arg_scope()):
_, end_points = inception.inception_v3(
x_generated, num_classes=FLAGS.num_classes, is_training=False)
predicted_labels = tf.argmax(end_points['Predictions'], 1)
predicted_logits = end_points['Logits']
disc_var_list = slim.get_model_variables()
# discriminator(resnet v2 50)
x_generated2 = tf.image.resize_bilinear(x_generated, [224, 224],
align_corners=False)
with slim.arg_scope(resnet_utils.resnet_arg_scope()):
_, end_points2 = resnet_v2.resnet_v2_50(
x_generated2, num_classes=FLAGS.num_classes, is_training=False)
predicted_labels2 = tf.argmax(end_points2['predictions'], 1)
predicted_logits2 = end_points2['predictions']
disc_var_list2 = slim.get_model_variables()[len(disc_var_list):]
# discriminator(resnet v2 152)
with slim.arg_scope(resnet_utils.resnet_arg_scope()):
_, end_points3 = resnet_v2.resnet_v2_152(
x_generated2, num_classes=FLAGS.num_classes, is_training=False)
predicted_labels3 = tf.argmax(end_points3['predictions'], 1)
predicted_logits3 = end_points3['predictions']
disc_var_list3 = slim.get_model_variables()[(len(disc_var_list) +
len(disc_var_list2)):]
# discriminator(resnet v2 101)
with slim.arg_scope(resnet_utils.resnet_arg_scope()):
_, end_points4 = resnet_v2.resnet_v2_101(
x_generated2, num_classes=FLAGS.num_classes, is_training=False)
predicted_labels4 = tf.argmax(end_points4['predictions'], 1)
predicted_logits4 = end_points4['predictions']
disc_var_list4 = slim.get_model_variables()[(len(disc_var_list) +
len(disc_var_list2) +
len(disc_var_list3)):]
# discriminator(inception v4)
with slim.arg_scope(inception.inception_v4_arg_scope()):
_, end_points5 = inception.inception_v4(
x_generated, num_classes=FLAGS.num_classes, is_training=False)
predicted_labels5 = tf.argmax(end_points5['Predictions'], 1)
predicted_logits5 = end_points['Logits']
disc_var_list5 = slim.get_model_variables()[(len(disc_var_list) +
len(disc_var_list2) +
len(disc_var_list3) +
len(disc_var_list4)):]
"""
# discriminator(vgg 19)
with slim.arg_scope(vgg.vgg_arg_scope()):
_, end_points3 = vgg.vgg_19(
x_generated2, num_classes=FLAGS.num_classes, is_training=False)
predicted_labels3 = tf.argmax(end_points3['vgg_19/fc8'], 1);
predicted_logits3 = end_points3['vgg_19/fc8'];
disc_var_list3=slim.get_model_variables()[(len(disc_var_list)+len(disc_var_list2)):];
"""
# loss and optimizer
gen_acc = tf.reduce_mean(
tf.cast(tf.equal(predicted_labels, tf.argmax(y_label, 1)),
tf.float32))
cross_entropy = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(logits=predicted_logits,
labels=y_label))
gen_acc2 = tf.reduce_mean(
tf.cast(tf.equal(predicted_labels2, tf.argmax(y_label, 1)),
tf.float32))
cross_entropy2 = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(logits=predicted_logits2,
labels=y_label))
gen_acc3 = tf.reduce_mean(
tf.cast(tf.equal(predicted_labels3, tf.argmax(y_label, 1)),
tf.float32))
cross_entropy3 = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(logits=predicted_logits3,
labels=y_label))
gen_acc4 = tf.reduce_mean(
tf.cast(tf.equal(predicted_labels4, tf.argmax(y_label, 1)),
tf.float32))
cross_entropy4 = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(logits=predicted_logits4,
labels=y_label))
gen_acc5 = tf.reduce_mean(
tf.cast(tf.equal(predicted_labels5, tf.argmax(y_label, 1)),
tf.float32))
cross_entropy5 = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(logits=predicted_logits5,
labels=y_label))
infi_norm = tf.reduce_mean(
tf.norm(tf.reshape(abs(x_data - x_generated),
[-1, FLAGS.img_size]),
ord=np.inf,
axis=1))
g_loss = -1 * cross_entropy - 1 * cross_entropy2 - 1 * cross_entropy3 - 1 * cross_entropy4 - 1 * cross_entropy5
optimizer = tf.train.AdamOptimizer(0.0001)
g_trainer = optimizer.minimize(g_loss, var_list=g_params)
# get the data and label
img_list = np.sort(glob.glob(FLAGS.input_folder + "*.png"))
total_data = np.zeros(
(len(img_list), FLAGS.img_height, FLAGS.img_width, 3), dtype=float)
for i in range(len(img_list)):
total_data[i] = imread(img_list[i], mode='RGB').astype(
np.float) / 255.0
total_data[i] = total_data[i] * 2.0 - 1.0
# 0~1 -> -1~1
val_data = np.copy(total_data[0])
f = open(FLAGS.label_folder + "true_label", "r")
total_label2 = np.array([i[:-1].split(",")[1] for i in f.readlines()],
dtype=int)
total_label = np.zeros((len(total_data), FLAGS.num_classes), dtype=int)
for i in range(len(total_data)):
total_label[i, total_label2[i]] = 1
val_label = np.copy(total_label[0])
# shuffle
total_idx = range(len(total_data))
np.random.shuffle(total_idx)
total_data = total_data[total_idx]
total_label = total_label[total_idx]
# Run computation
saver = tf.train.Saver(disc_var_list)
saver2 = tf.train.Saver(disc_var_list2)
saver3 = tf.train.Saver(disc_var_list3)
saver4 = tf.train.Saver(disc_var_list4)
saver5 = tf.train.Saver(disc_var_list5)
saver_gen = tf.train.Saver(g_params)
# initialization
init = tf.global_variables_initializer()
with tf.Session() as sess:
sess.run(init)
saver.restore(sess,
FLAGS.checkpoint_path + FLAGS.checkpoint_file_name)
saver2.restore(sess,
FLAGS.checkpoint_path + FLAGS.checkpoint_file_name2)
saver3.restore(sess,
FLAGS.checkpoint_path + FLAGS.checkpoint_file_name3)
saver4.restore(sess,
FLAGS.checkpoint_path + FLAGS.checkpoint_file_name4)
saver5.restore(sess,
FLAGS.checkpoint_path + FLAGS.checkpoint_file_name5)
# training
for i in range(FLAGS.max_epoch):
tr_infi = 0
tr_ce = 0
tr_gen_acc = 0
tr_ce2 = 0
tr_gen_acc2 = 0
tr_ce3 = 0
tr_gen_acc3 = 0
tr_ce4 = 0
tr_gen_acc4 = 0
tr_ce5 = 0
tr_gen_acc5 = 0
for j in range(len(total_data) / FLAGS.batch_size):
batch_data = total_data[j * FLAGS.batch_size:(j + 1) *
FLAGS.batch_size]
batch_label = total_label[j * FLAGS.batch_size:(j + 1) *
FLAGS.batch_size]
acc_p5, ce_p5, acc_p4, ce_p4, acc_p3, ce_p3, acc_p2, ce_p2, acc_p, ce_p, infi_p, _ = sess.run(
[
gen_acc5, cross_entropy5, gen_acc4, cross_entropy4,
gen_acc3, cross_entropy3, gen_acc2, cross_entropy2,
gen_acc, cross_entropy, infi_norm, g_trainer
],
feed_dict={
x_data: batch_data,
y_label: batch_label
})
tr_infi += infi_p
tr_ce += ce_p
tr_gen_acc += acc_p
tr_ce2 += ce_p2
tr_gen_acc2 += acc_p2
tr_ce3 += ce_p3
tr_gen_acc3 += acc_p3
tr_ce4 += ce_p4
tr_gen_acc4 += acc_p4
tr_ce5 += ce_p5
tr_gen_acc5 += acc_p5
print(
str(i + 1) + " Epoch InfiNorm:" + str(tr_infi / (j + 1)) +
",CE: " + str(tr_ce / (j + 1)) + ",Acc: " +
str(tr_gen_acc / (j + 1)) + ",CE2: " + str(tr_ce2 /
(j + 1)) +
",Acc2: " + str(tr_gen_acc2 / (j + 1)) + ",CE3: " +
str(tr_ce3 / (j + 1)) + ",Acc3: " + str(tr_gen_acc3 /
(j + 1)) +
",CE4: " + str(tr_ce4 / (j + 1)) + ",Acc4: " +
str(tr_gen_acc4 / (j + 1)) + ",CE5: " +
str(tr_ce5 / (j + 1)) + ",Acc5: " + str(tr_gen_acc5 /
(j + 1)))
total_idx = range(len(total_data))
np.random.shuffle(total_idx)
total_data = total_data[total_idx]
total_label = total_label[total_idx]
saver_gen.save(
sess, "my-models_iv3_rv250_rv2152_rv2101_iv4/my-model_" +
str(FLAGS.max_epsilon) + ".ckpt")
def mdr_net(inputs, args, image_size, is_training, reuse):
# mean subtraction normalization
inputs = inputs - [_R_MEAN, _G_MEAN, _B_MEAN]
if "resnet" in args.cnn_model:
# inputs has shape - Original: [batch, height, width, 3]
with slim.arg_scope(
resnet_utils.resnet_arg_scope(args.l2_regularizer, is_training,
args.batch_norm_decay,
args.batch_norm_epsilon)):
resnet = getattr(resnet_v2, args.cnn_model)
net, end_points = resnet(inputs,
multi_grid=args.multi_grid,
output_stride=args.output_stride,
global_pool=False,
num_classes=None,
reuse=reuse)
lower_level_features = end_points[args.cnn_model +
'/block1/unit_3/bottleneck_v2/conv1']
# low_level_features = end_points[args.cnn_model + 'block1/unit_2/bottleneck_v1/conv3']
elif "vgg" in args.cnn_model:
with slim.arg_scope(
vgg.vgg_arg_scope(args.l2_regularizer, is_training,
args.batch_norm_decay,
args.batch_norm_epsilon)):
net, end_points = vgg.vgg_16(inputs,
multi_grid=args.multi_grid,
output_stride=args.output_stride,
reuse=reuse)
lower_level_features = end_points[args.cnn_model + '/pool2']
else:
raise NameError("cnn_model must contain resnet or vgg!")
feature_map_size = [
int((sz - 1) / args.output_stride + 1) for sz in image_size
]
arg_sc = mdr_arg_scope(args.l2_regularizer, is_training,
args.batch_norm_decay, args.batch_norm_epsilon)
with slim.arg_scope(arg_sc):
with tf.variable_scope("MDR_Net", reuse=reuse):
encoder_output = atrous_spatial_pyramid_pooling(net,
"ASPP_layer",
is_training,
feature_map_size,
args.aspp_rates,
depth=256,
reuse=reuse)
with tf.variable_scope("decoder", reuse=reuse):
decoder_depth_1 = 256
if args.decoding_at_image_size:
decoder_depth_2 = 16
else:
decoder_depth_2 = 1
lower_level_feature_depth = 48
with tf.variable_scope("lower_level_features"):
lower_level_features = slim.conv2d(
lower_level_features,
lower_level_feature_depth, [1, 1],
stride=1,
scope='conv_1x1')
lower_level_features_size = tf.shape(
lower_level_features)[1:3]
with tf.variable_scope("upsampling_logits_1"):
net = tf.image.resize_bilinear(encoder_output,
lower_level_features_size,
name='upsample_1',
align_corners=True)
net = tf.concat([net, lower_level_features],
axis=3,
name='concat')
num_convs = 2
decoder_features = slim.repeat(net,
num_convs,
slim.conv2d,
decoder_depth_1,
3,
scope='decoder_conv1')
with tf.variable_scope("upsampling_logits_2"):
if args.decoding_at_image_size:
decoder_features = slim.conv2d(decoder_features,
decoder_depth_2, [3, 3],
scope='decoder_conv2')
net = tf.image.resize_bilinear(decoder_features,
image_size,
name='upsample_2',
align_corners=True)
logits = slim.conv2d(net,
1, [1, 1],
activation_fn=None,
normalizer_fn=None,
scope='conv_logits')
else:
decoder_features = slim.conv2d(decoder_features,
decoder_depth_2, [1, 1],
activation_fn=None,
normalizer_fn=None,
scope='conv_logits')
logits = tf.image.resize_bilinear(decoder_features,
image_size,
name='upsample_2',
align_corners=True)
return logits
def main(_):
batch_shape = [FLAGS.batch_size, FLAGS.image_height, FLAGS.image_width, 3]
num_classes = 1001
# max_epsilon over checking
# get original images
origin_img_list=np.sort(glob.glob(FLAGS.origin_img_dir+"*.png"));
origin_imgs=np.zeros((len(origin_img_list),FLAGS.image_height,FLAGS.image_width,3),dtype=float);
for i in range(len(origin_img_list)):
origin_imgs[i]=imread(origin_img_list[i],mode='RGB').astype(np.float);
# get adv images
adv_img_list=np.sort(glob.glob(FLAGS.input_dir+"*.png"));
adv_imgs=np.zeros((len(adv_img_list),FLAGS.image_height,FLAGS.image_width,3),dtype=float);
for i in range(len(adv_img_list)):
adv_imgs[i]=imread(adv_img_list[i],mode='RGB').astype(np.float);
epsilon_list=np.linalg.norm(np.reshape(abs(origin_imgs-adv_imgs),[-1,FLAGS.image_height*FLAGS.image_width*3]),ord=np.inf,axis=1);
#print(epsilon_list);exit(1);
over_epsilon_list=np.zeros((len(origin_img_list),2),dtype=object);
cnt=0;
for i in range(len(origin_img_list)):
file_name=origin_img_list[i].split("/")[-1];
file_name=file_name.split(".")[0];
over_epsilon_list[i,0]=file_name;
if(epsilon_list[i]>FLAGS.max_epsilon):
over_epsilon_list[i,1]="1";
cnt+=1;
tf.logging.set_verbosity(tf.logging.INFO)
with tf.Graph().as_default():
# Prepare graph
x_input = tf.placeholder(tf.float32, shape=batch_shape)
if(FLAGS.checkpoint_file_name=="inception_v3.ckpt"):
with slim.arg_scope(inception.inception_v3_arg_scope()):
_, end_points = inception.inception_v3(
x_input, num_classes=num_classes, is_training=False)
predicted_labels = tf.argmax(end_points['Predictions'], 1)
elif(FLAGS.checkpoint_file_name=="inception_v4.ckpt"):
with slim.arg_scope(inception.inception_v4_arg_scope()):
_, end_points = inception.inception_v4(
x_input, num_classes=num_classes, is_training=False)
predicted_labels = tf.argmax(end_points['Predictions'], 1)
elif(FLAGS.checkpoint_file_name=="inception_resnet_v2_2016_08_30.ckpt"):
with slim.arg_scope(inception.inception_resnet_v2_arg_scope()):
_, end_points = inception.inception_resnet_v2(
x_input, num_classes=num_classes, is_training=False)
predicted_labels = tf.argmax(end_points['Predictions'], 1)
elif(FLAGS.checkpoint_file_name=="resnet_v2_101.ckpt"):
x_input2 = tf.image.resize_bilinear(x_input,[224,224],align_corners=False);
with slim.arg_scope(resnet_utils.resnet_arg_scope()):
_, end_points = resnet_v2.resnet_v2_101(
x_input2, num_classes=num_classes, is_training=False)
predicted_labels = tf.argmax(end_points['predictions'], 1)
elif(FLAGS.checkpoint_file_name=="resnet_v2_50.ckpt"):
x_input2 = tf.image.resize_bilinear(x_input,[224,224],align_corners=False);
with slim.arg_scope(resnet_utils.resnet_arg_scope()):
_, end_points = resnet_v2.resnet_v2_50(
x_input2, num_classes=num_classes, is_training=False)
predicted_labels = tf.argmax(end_points['predictions'], 1)
elif(FLAGS.checkpoint_file_name=="resnet_v2_152.ckpt"):
x_input2 = tf.image.resize_bilinear(x_input,[224,224],align_corners=False);
with slim.arg_scope(resnet_utils.resnet_arg_scope()):
_, end_points = resnet_v2.resnet_v2_152(
x_input2, num_classes=num_classes, is_training=False)
predicted_labels = tf.argmax(end_points['predictions'], 1)
elif(FLAGS.checkpoint_file_name=="inception_v1.ckpt"):
x_input2 = tf.image.resize_bilinear(x_input,[224,224],align_corners=False);
with slim.arg_scope(inception.inception_v1_arg_scope()):
_, end_points = inception.inception_v1(
x_input2, num_classes=num_classes, is_training=False)
predicted_labels = tf.argmax(end_points['Predictions'], 1)
elif(FLAGS.checkpoint_file_name=="inception_v2.ckpt"):
x_input2 = tf.image.resize_bilinear(x_input,[224,224],align_corners=False);
with slim.arg_scope(inception.inception_v2_arg_scope()):
_, end_points = inception.inception_v2(
x_input2, num_classes=num_classes, is_training=False)
predicted_labels = tf.argmax(end_points['Predictions'], 1)
# Resnet v1 and vgg are not working now
elif(FLAGS.checkpoint_file_name=="vgg_16.ckpt"):
x_input_list=tf.unstack(x_input,FLAGS.batch_size,0);
for i in range(FLAGS.batch_size):
x_input_list[i]=vgg_preprocessing.preprocess_image(x_input_list[i],224,224);
x_input2=tf.stack(x_input_list,0);
with slim.arg_scope(vgg.vgg_arg_scope()):
_, end_points = vgg.vgg_16(
x_input2, num_classes=num_classes-1, is_training=False)
predicted_labels = tf.argmax(end_points['vgg_16/fc8'], 1)+1
elif(FLAGS.checkpoint_file_name=="vgg_19.ckpt"):
x_input_list=tf.unstack(x_input,FLAGS.batch_size,0);
for i in range(FLAGS.batch_size):
x_input_list[i]=vgg_preprocessing.preprocess_image(x_input_list[i],224,224);
x_input2=tf.stack(x_input_list,0);
with slim.arg_scope(vgg.vgg_arg_scope()):
_, end_points = vgg.vgg_19(
x_input2, num_classes=num_classes-1, is_training=False)
predicted_labels = tf.argmax(end_points['vgg_19/fc8'], 1)+1
elif(FLAGS.checkpoint_file_name=="resnet_v1_50.ckpt"):
x_input_list=tf.unstack(x_input,FLAGS.batch_size,0);
for i in range(FLAGS.batch_size):
x_input_list[i]=vgg_preprocessing.preprocess_image(x_input_list[i],224,224);
x_input2=tf.stack(x_input_list,0);
with slim.arg_scope(resnet_utils.resnet_arg_scope()):
_, end_points = resnet_v1.resnet_v1_50(
x_input, num_classes=num_classes-1, is_training=False)
predicted_labels = tf.argmax(end_points['predictions'], 1)+1
elif(FLAGS.checkpoint_file_name=="resnet_v1_101.ckpt"):
x_input_list=tf.unstack(x_input,FLAGS.batch_size,0);
for i in range(FLAGS.batch_size):
x_input_list[i]=vgg_preprocessing.preprocess_image(x_input_list[i],224,224);
x_input2=tf.stack(x_input_list,0);
with slim.arg_scope(resnet_utils.resnet_arg_scope()):
_, end_points = resnet_v1.resnet_v1_101(
x_input2, num_classes=num_classes-1, is_training=False)
predicted_labels = tf.argmax(end_points['predictions'], 1)+1
elif(FLAGS.checkpoint_file_name=="resnet_v1_152.ckpt"):
x_input_list=tf.unstack(x_input,FLAGS.batch_size,0);
for i in range(FLAGS.batch_size):
x_input_list[i]=vgg_preprocessing.preprocess_image(x_input_list[i],224,224);
x_input2=tf.stack(x_input_list,0);
with slim.arg_scope(resnet_utils.resnet_arg_scope()):
_, end_points = resnet_v1.resnet_v1_152(
x_input2, num_classes=num_classes-1, is_training=False)
predicted_labels = tf.argmax(end_points['predictions'], 1)+1
# Run computation
saver = tf.train.Saver(slim.get_model_variables())
session_creator = tf.train.ChiefSessionCreator(
scaffold=tf.train.Scaffold(saver=saver),
checkpoint_filename_with_path=FLAGS.checkpoint_path+FLAGS.checkpoint_file_name,
master=FLAGS.master)
f=open(FLAGS.true_label,"r");
t_label_list=np.array([i[:-1].split(",") for i in f.readlines()]);
score=0;
with tf.train.MonitoredSession(session_creator=session_creator) as sess:
with tf.gfile.Open(FLAGS.output_file, 'w') as out_file:
for filenames, images in load_images(FLAGS.input_dir, batch_shape):
labels = sess.run(predicted_labels, feed_dict={x_input: images})
for filename, label in zip(filenames, labels):
f_name=filename.split(".")[0];
t_label=int(t_label_list[t_label_list[:,0]==f_name,1][0]);
if(t_label!=label):
if(over_epsilon_list[over_epsilon_list[:,0]==f_name,1]!="1"):
score+=1;
#out_file.write('{0},{1}\n'.format(filename, label))
print("Over max epsilon#: "+str(cnt));
print(str(FLAGS.max_epsilon)+" max epsilon Score: "+str(score));
def testStridingLastUnitVsSubsampleBlockEnd(self):
"""Compares subsampling at the block's last unit or block's end.
Makes sure that the final output is the same when we use a stride at the
last unit of a block vs. we subsample activations at the end of a block.
"""
block = resnet_v1.resnet_v1_block
blocks = [
block('block1', base_depth=1, num_units=2, stride=2),
block('block2', base_depth=2, num_units=2, stride=2),
block('block3', base_depth=4, num_units=2, stride=2),
block('block4', base_depth=8, num_units=2, stride=1),
]
# Test both odd and even input dimensions.
height = 30
width = 31
with slim.arg_scope(resnet_utils.resnet_arg_scope()):
with slim.arg_scope([slim.batch_norm], is_training=False):
for output_stride in [1, 2, 4, 8, None]:
with tf.Graph().as_default():
with self.test_session() as sess:
tf.set_random_seed(0)
inputs = create_test_input(1, height, width, 3)
# Subsampling at the last unit of the block.
output = resnet_utils.stack_blocks_dense(
inputs,
blocks,
output_stride,
store_non_strided_activations=False,
outputs_collections='output')
output_end_points = slim.utils.convert_collection_to_dict(
'output')
# Make the two networks use the same weights.
tf.get_variable_scope().reuse_variables()
# Subsample activations at the end of the blocks.
expected = resnet_utils.stack_blocks_dense(
inputs,
blocks,
output_stride,
store_non_strided_activations=True,
outputs_collections='expected')
expected_end_points = slim.utils.convert_collection_to_dict(
'expected')
sess.run(tf.global_variables_initializer())
# Make sure that the final output is the same.
output, expected = sess.run([output, expected])
self.assertAllClose(output,
expected,
atol=1e-4,
rtol=1e-4)
# Make sure that intermediate block activations in
# output_end_points are subsampled versions of the corresponding
# ones in expected_end_points.
for i, block in enumerate(blocks[:-1:]):
output = output_end_points[block.scope]
expected = expected_end_points[block.scope]
atrous_activated = (output_stride is not None
and 2**i >= output_stride)
if not atrous_activated:
expected = resnet_utils.subsample(
expected, 2)
output, expected = sess.run([output, expected])
self.assertAllClose(output,
expected,
atol=1e-4,
rtol=1e-4)
def resnet_arg_scope(weight_decay=0.0001):
print_red("Patching resnet_v2 arg_scope when training from scratch")
return resnet_utils.resnet_arg_scope(weight_decay=weight_decay,
batch_norm_decay=0.9,
batch_norm_epsilon=5e-4,
batch_norm_scale=False)
def train():
eps=2.0*float(FLAGS.max_epsilon)/256.0;
tf.logging.set_verbosity(tf.logging.INFO);
with tf.Graph().as_default():
# Design architecture
# input
x_data = tf.placeholder(tf.float32, [None, FLAGS.img_height, FLAGS.img_width,3], name="x_data")
y_label = tf.placeholder(tf.float32, [None, FLAGS.num_classes], name="y_label")
# generator
x_generated, g_params = build_generator(x_data,FLAGS);
x_generated = x_generated * eps;
x_generated = x_data + x_generated;
# discriminator(inception v3)
with slim.arg_scope(inception.inception_v3_arg_scope()):
_, end_points = inception.inception_v3(
x_generated, num_classes=FLAGS.num_classes, is_training=False)
predicted_labels = tf.argmax(end_points['Predictions'], 1);
predicted_logits = end_points['Logits'];
disc_var_list=slim.get_model_variables();
# discriminator(resnet v2 50)
x_generated2=tf.image.resize_bilinear(x_generated,[224,224],align_corners=False);
with slim.arg_scope(resnet_utils.resnet_arg_scope()):
_, end_points2 = resnet_v2.resnet_v2_50(
x_generated2, num_classes=FLAGS.num_classes, is_training=False)
predicted_labels2 = tf.argmax(end_points2['predictions'], 1);
predicted_logits2 = end_points2['predictions'];
disc_var_list2=slim.get_model_variables()[len(disc_var_list):];
# discriminator(resnet v2 152)
with slim.arg_scope(resnet_utils.resnet_arg_scope()):
_, end_points3 = resnet_v2.resnet_v2_152(
x_generated2, num_classes=FLAGS.num_classes, is_training=False)
predicted_labels3 = tf.argmax(end_points3['predictions'], 1);
predicted_logits3 = end_points3['predictions'];
disc_var_list3=slim.get_model_variables()[(len(disc_var_list)+len(disc_var_list2)):];
# average
predicted_prob_avg = (end_points['Predictions']+end_points2['predictions']+end_points3['predictions'])/5.0;
predicted_labels_avg = tf.argmax((end_points['Predictions']+end_points2['predictions']+end_points3['predictions'])/5.0, 1);
# loss and optimizer
gen_acc=tf.reduce_mean(tf.cast(tf.equal(predicted_labels,tf.argmax(y_label,1)),tf.float32));
cross_entropy=tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=predicted_logits, labels=y_label));
gen_acc2=tf.reduce_mean(tf.cast(tf.equal(predicted_labels2,tf.argmax(y_label,1)),tf.float32));
cross_entropy2=tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=predicted_logits2, labels=y_label));
gen_acc3=tf.reduce_mean(tf.cast(tf.equal(predicted_labels3,tf.argmax(y_label,1)),tf.float32));
cross_entropy3=tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=predicted_logits3, labels=y_label));
gen_acc_avg=tf.reduce_mean(tf.cast(tf.equal(predicted_labels_avg,tf.argmax(y_label,1)),tf.float32));
cross_entropy_avg=tf.reduce_mean(-tf.reduce_sum(y_label*tf.log(predicted_prob_avg),1));
infi_norm=tf.reduce_mean(tf.norm(tf.reshape(abs(x_data-x_generated),[-1,FLAGS.img_size]),ord=np.inf,axis=1));
g_loss=-1*cross_entropy_avg;
optimizer = tf.train.AdamOptimizer(0.0001)
g_trainer = optimizer.minimize(g_loss, var_list=g_params)
# get the data and label
img_list=np.sort(glob.glob(FLAGS.input_folder+"*.png"));
total_data=np.zeros((len(img_list),FLAGS.img_height,FLAGS.img_width,3),dtype=float);
for i in range(len(img_list)):
total_data[i]=imread(img_list[i],mode='RGB').astype(np.float) / 255.0;
total_data[i]=total_data[i]*2.0-1.0; # 0~1 -> -1~1
val_data=np.copy(total_data[0]);
f=open(FLAGS.label_folder+"true_label","r");
total_label2=np.array([i[:-1].split(",")[1] for i in f.readlines()],dtype=int);
total_label=np.zeros((len(total_data),FLAGS.num_classes),dtype=int);
for i in range(len(total_data)):
total_label[i,total_label2[i]]=1;
val_label=np.copy(total_label[0]);
# shuffle
total_idx=range(len(total_data)); np.random.shuffle(total_idx);
total_data=total_data[total_idx];total_label=total_label[total_idx];
# Run computation
saver = tf.train.Saver(disc_var_list);
saver2 = tf.train.Saver(disc_var_list2);
saver3 = tf.train.Saver(disc_var_list3);
saver_gen = tf.train.Saver(g_params);
# initialization
init = tf.global_variables_initializer();
with tf.Session() as sess:
sess.run(init)
saver.restore(sess,FLAGS.checkpoint_path+FLAGS.checkpoint_file_name);
saver2.restore(sess,FLAGS.checkpoint_path+FLAGS.checkpoint_file_name2);
saver3.restore(sess,FLAGS.checkpoint_path+FLAGS.checkpoint_file_name3);
# training
for i in range(FLAGS.max_epoch):
tr_infi=0;
tr_ce=0;
tr_gen_acc=0;
tr_ce2=0;
tr_gen_acc2=0;
tr_ce3=0;
tr_gen_acc3=0;
tr_ce_avg=0;
tr_gen_acc_avg=0;
for j in range(len(total_data) / FLAGS.batch_size):
batch_data=total_data[j*FLAGS.batch_size:(j+1)*FLAGS.batch_size];
batch_label=total_label[j*FLAGS.batch_size:(j+1)*FLAGS.batch_size];
acc_p_a,ce_p_a,acc_p3,ce_p3,acc_p2,ce_p2,acc_p,ce_p,infi_p,_=sess.run([gen_acc_avg,cross_entropy_avg,gen_acc3,cross_entropy3,gen_acc2,cross_entropy2,gen_acc,cross_entropy,infi_norm,g_trainer],feed_dict={x_data: batch_data, y_label: batch_label});
tr_infi+=infi_p;
tr_ce+=ce_p;
tr_gen_acc+=acc_p;
tr_ce2+=ce_p2;
tr_gen_acc2+=acc_p2;
tr_ce3+=ce_p3;
tr_gen_acc3+=acc_p3;
tr_ce_avg+=ce_p_a;
tr_gen_acc_avg+=acc_p_a;
print(str(i+1)+" Epoch InfiNorm:"+str(tr_infi/(j+1))+",CE: "+str(tr_ce/(j+1))+",Acc: "+str(tr_gen_acc/(j+1))+",CE2: "+str(tr_ce2/(j+1))+",Acc2: "+str(tr_gen_acc2/(j+1))+",CE3: "+str(tr_ce3/(j+1))+",Acc3: "+str(tr_gen_acc3/(j+1))+",Acc_avg: "+str(tr_gen_acc_avg/(j+1))+",CE_avg: "+str(tr_ce_avg/(j+1)));
total_idx=range(len(total_data)); np.random.shuffle(total_idx);
total_data=total_data[total_idx];total_label=total_label[total_idx];
saver_gen.save(sess,"my-models_iv3_rv250_rv2152_avg2/my-model_"+str(FLAGS.max_epsilon)+".ckpt");
def _extract_proposal_features(self, preprocessed_inputs, scope):
"""Extracts first stage RPN features.
Args:
preprocessed_inputs: A [batch, height, width, channels] float32 tensor
representing a batch of images.
scope: A scope name.
Returns:
rpn_feature_map: A tensor with shape [batch, height, width, depth]
activations: A dictionary mapping feature extractor tensor names to
tensors
Raises:
InvalidArgumentError: If the spatial size of `preprocessed_inputs`
(height or width) is less than 33.
ValueError: If the created network is missing the required activation.
"""
if len(preprocessed_inputs.get_shape().as_list()) != 4:
raise ValueError('`preprocessed_inputs` must be 4 dimensional, got a '
'tensor of shape %s' % preprocessed_inputs.get_shape())
shape_assert = tf.Assert(
tf.logical_and(
tf.greater_equal(tf.shape(preprocessed_inputs)[1], 33),
tf.greater_equal(tf.shape(preprocessed_inputs)[2], 33)),
['image size must at least be 33 in both height and width.'])
with tf.control_dependencies([shape_assert]):
# Disables batchnorm for fine-tuning with smaller batch sizes.
# TODO(chensun): Figure out if it is needed when image
# batch size is bigger.
with slim.arg_scope(
resnet_utils.resnet_arg_scope(
batch_norm_epsilon=1e-5,
batch_norm_scale=True,
weight_decay=self._weight_decay)):
with tf.variable_scope(
self._architecture, reuse=self._reuse_weights) as var_scope:
_, image_features = self._resnet_model(
preprocessed_inputs,
num_classes=None,
is_training=self._train_batch_norm,
global_pool=False,
output_stride=self._first_stage_features_stride,
spatial_squeeze=False,
scope=var_scope)
block_name = ['block3', 'block4'] if self._first_stage_features_outblock == 4 else ['block2', 'block3']
features_depth_output = 512
image_features = self._filter_features(image_features)
last_feature_map = image_features[block_name[1]]
last_feature_map = slim.conv2d(
last_feature_map,
num_outputs=features_depth_output,
kernel_size=[1, 1],
stride=1,
activation_fn=None,
normalizer_fn=None,
padding='SAME',
scope=block_name[1]+'_side')
second_last_feature_map = image_features[block_name[0]]
top_down = tf.image.resize_images(last_feature_map, tf.shape(second_last_feature_map)[1:3])
residual = slim.conv2d(
second_last_feature_map,
num_outputs =features_depth_output,
kernel_size=[1, 1],
stride = 1,
activation_fn=None,
normalizer_fn=None,
padding='SAME',
scope=block_name[0]+'_side')
top_down = 0.5 * top_down + 0.5 * residual
# top_down_up = tf.image.resize_images(top_down, tf.shape(image_features['block1'])[1:3])
# residual_1 = slim.conv2d(
# image_features['block1'],
# num_outputs = 256,
# kernel_size=[1, 1],
# stride = 1,
# activation_fn=None,
# normalizer_fn=None,
# padding='SAME',
# scope='block1_side')
# top_down = 0.5 * top_down_up + 0.5 * residual_1
return top_down, image_features
outfile = 'answers.csv'
string = ''
string += 'id,landmarks\n'
for filename in os.listdir('data/test'):
image = os.path.join('data/test', filename)
f = open(image)
image_string = f.read()
image = tf.image.decode_jpeg(image_string, channels=3)
processed_image = crop_and_resize(image,
image_size,
image_size,
is_training=False)
processed_images = tf.expand_dims(processed_image, 0)
with slim.arg_scope(resnet_utils.resnet_arg_scope()):
logits, _ = resnet_v2.resnet_v2_50(processed_images,
num_classes=14951,
is_training=False)
probabilities = tf.nn.softmax(logits)
init_fn = slim.assign_from_checkpoint_fn(
os.path.join(checkpoints_dir, 'model.ckpt-31909'),
slim.get_model_variables('resnet_v2_50'))
with tf.Session() as sess:
init_fn(sess)
np_image, network_input, probabilities = sess.run(
[image, processed_image, probabilities])
probabilities = probabilities[0, 0:]
sorted_inds = [
def train():
eps=2.0*float(FLAGS.max_epsilon)/256.0;
tf.logging.set_verbosity(tf.logging.INFO);
with tf.Graph().as_default():
# Design architecture
# input
is_training = tf.placeholder(tf.bool);
x_data = tf.placeholder(tf.float32, [None, FLAGS.img_height, FLAGS.img_width,3], name="x_data")
y_label = tf.placeholder(tf.float32, [None, FLAGS.num_classes], name="y_label")
y_label_ll = tf.placeholder(tf.float32, [None, FLAGS.num_classes], name="y_label_ll")
lam = tf.placeholder(tf.float32, [], name="lambda");
# generator
x_generated, g_params, bn_var_num = build_generator(x_data,is_training,FLAGS);
x_generated = x_generated * eps;
x_generated = x_data + x_generated;
# discriminator(inception v3)
with slim.arg_scope(inception.inception_v3_arg_scope()):
_, end_points = inception.inception_v3(
x_generated, num_classes=FLAGS.num_classes, is_training=False)
predicted_labels = tf.argmax(end_points['Predictions'], 1);
predicted_logits = end_points['Logits'];
disc_var_list=slim.get_model_variables()[bn_var_num:];
# discriminator(resnet v2 50)
x_generated2=tf.image.resize_bilinear(x_generated,[224,224],align_corners=False);
with slim.arg_scope(resnet_utils.resnet_arg_scope()):
_, end_points2 = resnet_v2.resnet_v2_50(
x_generated2, num_classes=FLAGS.num_classes, is_training=False)
predicted_labels2 = tf.argmax(end_points2['predictions'], 1);
predicted_logits2 = end_points2['predictions'];
disc_var_list2=slim.get_model_variables()[(bn_var_num+len(disc_var_list)):];
# discriminator(resnet v2 152)
with slim.arg_scope(resnet_utils.resnet_arg_scope()):
_, end_points3 = resnet_v2.resnet_v2_152(
x_generated2, num_classes=FLAGS.num_classes, is_training=False)
predicted_labels3 = tf.argmax(end_points3['predictions'], 1);
predicted_logits3 = end_points3['predictions'];
disc_var_list3=slim.get_model_variables()[(bn_var_num+len(disc_var_list)+len(disc_var_list2)):];
# discriminator(resnet v2 101)
with slim.arg_scope(resnet_utils.resnet_arg_scope()):
_, end_points4 = resnet_v2.resnet_v2_101(
x_generated2, num_classes=FLAGS.num_classes, is_training=False)
predicted_labels4 = tf.argmax(end_points4['predictions'], 1);
predicted_logits4 = end_points4['predictions'];
disc_var_list4=slim.get_model_variables()[(bn_var_num+len(disc_var_list)+len(disc_var_list2)+len(disc_var_list3)):];
# discriminator(inception v4)
with slim.arg_scope(inception.inception_v4_arg_scope()):
_, end_points5 = inception.inception_v4(
x_generated, num_classes=FLAGS.num_classes, is_training=False)
predicted_labels5 = tf.argmax(end_points5['Predictions'], 1);
predicted_logits5 = end_points['Logits'];
disc_var_list5=slim.get_model_variables()[(bn_var_num+len(disc_var_list)+len(disc_var_list2)+len(disc_var_list3)+len(disc_var_list4)):];
# average
predicted_prob_avg = (end_points['Predictions']+end_points2['predictions']+end_points3['predictions']+end_points4['predictions']+end_points5['Predictions'])/5.0;
predicted_labels_avg = tf.argmax((end_points['Predictions']+end_points2['predictions']+end_points3['predictions']+end_points4['predictions']+end_points5['Predictions'])/5.0, 1);
# loss and optimizer
gen_acc=tf.reduce_mean(tf.cast(tf.equal(predicted_labels_avg,tf.argmax(y_label,1)),tf.float32));
cross_entropy=tf.reduce_mean(-tf.reduce_sum(y_label*tf.log(predicted_prob_avg),1));
cross_entropy_ll=tf.reduce_mean(-tf.reduce_sum(y_label_ll*tf.log(predicted_prob_avg),1));
infi_norm=tf.reduce_mean(tf.norm(tf.reshape(abs(x_data-x_generated),[-1,FLAGS.img_size]),ord=np.inf,axis=1));
g_loss=-1*cross_entropy+cross_entropy_ll;
optimizer = tf.train.AdamOptimizer(0.0001)
g_trainer = optimizer.minimize(g_loss, var_list=g_params)
# get the data and label
img_list=np.sort(glob.glob(FLAGS.input_folder+"*.png"));
total_data=np.zeros((len(img_list),FLAGS.img_height,FLAGS.img_width,3),dtype=float);
for i in range(len(img_list)):
total_data[i]=imread(img_list[i],mode='RGB').astype(np.float) / 255.0;
total_data[i]=total_data[i]*2.0-1.0; # 0~1 -> -1~1
val_data=np.copy(total_data[0]);
f=open(FLAGS.label_folder+"true_label","r");
total_label2=np.array([i[:-1].split(",")[1] for i in f.readlines()],dtype=int);
total_label=np.zeros((len(total_data),FLAGS.num_classes),dtype=int);
for i in range(len(total_data)):
total_label[i,total_label2[i]]=1;
f=open("logits","r");
total_logits=np.array([i[:-1].split(",") for i in f.readlines()],dtype=float);
total_label_ll=np.zeros((len(total_data),FLAGS.num_classes),dtype=int);
for i in range(len(total_data)):
#total_logits[i,total_label2[i]]=0.0;
target_idx=np.argmin(total_logits[i]);
total_label_ll[i,target_idx]=1;
val_label=np.copy(total_label[0]);
# shuffle
total_idx=range(len(total_data)); np.random.shuffle(total_idx);
total_data=total_data[total_idx];total_label=total_label[total_idx];
# Run computation
saver = tf.train.Saver(disc_var_list);
saver2 = tf.train.Saver(disc_var_list2);
saver3 = tf.train.Saver(disc_var_list3);
saver4 = tf.train.Saver(disc_var_list4);
saver5 = tf.train.Saver(disc_var_list5);
saver_gen = tf.train.Saver(g_params);
"""
session_creator = tf.train.ChiefSessionCreator(
scaffold=tf.train.Scaffold(saver=saver),
checkpoint_filename_with_path=FLAGS.checkpoint_path,
master=FLAGS.master)
"""
# initialization
init = tf.global_variables_initializer();
with tf.Session() as sess:
#with tf.train.MonitoredSession() as sess:
sess.run(init)
saver.restore(sess,FLAGS.checkpoint_path+"inception_v3.ckpt");
saver2.restore(sess,FLAGS.checkpoint_path+"resnet_v2_50.ckpt");
saver3.restore(sess,FLAGS.checkpoint_path+"resnet_v2_152.ckpt");
saver4.restore(sess,FLAGS.checkpoint_path+"resnet_v2_101.ckpt");
saver5.restore(sess,FLAGS.checkpoint_path+"inception_v4.ckpt");
# tf board
tf.summary.scalar('reverse_cross_entropy',cross_entropy);
tf.summary.scalar('training_accuracy',gen_acc);
merged = tf.summary.merge_all()
train_writer = tf.summary.FileWriter('/tmp/nips17/attack_gan/learn_gan5_avg2_ll_'+str(FLAGS.max_epsilon), sess.graph)
# training
for i in range(FLAGS.max_epoch):
if(i>100):
lam_value=1.0;
else:
lam_value=1.0;
tr_ce=0;
tr_infi=0;
tr_gen_acc=0;
for j in range(len(total_data) / FLAGS.batch_size):
batch_data=total_data[j*FLAGS.batch_size:(j+1)*FLAGS.batch_size];
batch_label=total_label[j*FLAGS.batch_size:(j+1)*FLAGS.batch_size];
batch_label_ll=total_label_ll[j*FLAGS.batch_size:(j+1)*FLAGS.batch_size];
summary,tr_gen_acc_part,tr_ce_part,tr_infi_part,_=sess.run([merged,gen_acc,cross_entropy,infi_norm,g_trainer],feed_dict={is_training:True,x_data: batch_data, y_label: batch_label,y_label_ll:batch_label_ll,lam:lam_value});
tr_ce+=tr_ce_part;
tr_infi+=tr_infi_part;
tr_gen_acc+=tr_gen_acc_part;
train_writer.add_summary(summary,i*len(total_data)+j*FLAGS.batch_size);
print(str(i+1)+" Epoch Training Cross Entropy: "+str(tr_ce/(j+1))+", Infinity Norm: "+str(tr_infi/(j+1))+",Gen Acc: "+str(tr_gen_acc/(j+1)));
total_idx=range(len(total_data)); np.random.shuffle(total_idx);
total_data=total_data[total_idx];total_label=total_label[total_idx];
saver_gen.save(sess,"mark3_iv3_rv250_rv2152_rv2101_iv4_avg2/my-model_"+str(FLAGS.max_epsilon)+".ckpt");
