def net_fatory(net_name, inputs, train_model, FC=False):
if net_name == 'vgg_16':
with slim.arg_scope(vgg.vgg_arg_scope()):
net, end_points = vgg.vgg_16(inputs,
num_classes=None,
is_training=train_model,
fc_flage=FC)
elif net_name == 'vgg_19':
with slim.arg_scope(vgg.vgg_arg_scope()):
net, end_points = vgg.vgg_19(inputs,
num_classes=None,
is_training=train_model,
fc_flage=FC)
elif net_name == 'resnet_v2_50':
with slim.arg_scope(resnet_arg_scope()):
net, end_points = resnet_v2.resnet_v2_50(inputs=inputs,
num_classes=None,
is_training=train_model,
global_pool=False)
elif net_name == 'resnet_v2_152':
with slim.arg_scope(resnet_arg_scope()):
net, end_points = resnet_v2.resnet_v2_152(inputs=inputs,
num_classes=None,
is_training=train_model,
global_pool=False)
return net, end_points
def _image_to_head(self, is_training, reuse=None):
fix = cfg.RESNET.FIXED_BLOCKS
assert (0 <= fix <= 3)
# Now the base is always fixed during training
with slim.arg_scope(resnet_arg_scope(is_training=False)):
net_conv = self._build_base()
print("Fixing %s blocks." % cfg.RESNET.FIXED_BLOCKS)
with slim.arg_scope(resnet_arg_scope()):
c1, _ = resnet_v2(net_conv,
[self._blocks[0]],
is_training=0 >= fix and is_training,
global_pool=False,
include_root_block=False,
reuse=reuse,
scope=self._resnet_scope)
if 0 >= fix and is_training:
print("training block 1")
c2, _ = resnet_v2(c1,
[self._blocks[1]],
is_training=1 >= fix and is_training,
global_pool=False,
include_root_block=False,
reuse=reuse,
scope=self._resnet_scope)
if 1 >= fix and is_training:
print("training block 2")
c3, _ = resnet_v2(c2,
[self._blocks[2]],
is_training=2 >= fix and is_training,
global_pool=False,
include_root_block=False,
reuse=reuse,
scope=self._resnet_scope)
if 2 >= fix and is_training:
print("training block 3")
c4, _ = resnet_v2(c3,
[self._blocks[3]],
is_training=is_training,
global_pool=False,
include_root_block=False,
postnorm=True,
reuse=reuse,
scope=self._resnet_scope)
self._layers["c1"] = c1
self._layers["c2"] = c2
self._layers["c3"] = c3
self._layers["c4"] = c4
return c4
def testEndPointsV2(self):
"""Test the end points of a tiny v2 bottleneck network."""
blocks = [
resnet_v2.resnet_v2_block('block1',
base_depth=1,
num_units=2,
stride=2),
resnet_v2.resnet_v2_block('block2',
base_depth=2,
num_units=2,
stride=1),
]
inputs = create_test_input(2, 32, 16, 3)
with slim.arg_scope(resnet_utils.resnet_arg_scope()):
_, end_points = self._resnet_plain(inputs, blocks, scope='tiny')
expected = [
'tiny/block1/unit_1/bottleneck_v2/shortcut',
'tiny/block1/unit_1/bottleneck_v2/conv1',
'tiny/block1/unit_1/bottleneck_v2/conv2',
'tiny/block1/unit_1/bottleneck_v2/conv3',
'tiny/block1/unit_2/bottleneck_v2/conv1',
'tiny/block1/unit_2/bottleneck_v2/conv2',
'tiny/block1/unit_2/bottleneck_v2/conv3',
'tiny/block2/unit_1/bottleneck_v2/shortcut',
'tiny/block2/unit_1/bottleneck_v2/conv1',
'tiny/block2/unit_1/bottleneck_v2/conv2',
'tiny/block2/unit_1/bottleneck_v2/conv3',
'tiny/block2/unit_2/bottleneck_v2/conv1',
'tiny/block2/unit_2/bottleneck_v2/conv2',
'tiny/block2/unit_2/bottleneck_v2/conv3'
]
self.assertItemsEqual(expected, end_points.keys())
def _select_model(self, model, class_num, root_pretrain):
self.input = tf.placeholder(tf.float32, [None, self.image_size[1], self.image_size[0], 3], name='image_input')
#self.labels = tf.placeholder(tf.int64, [None]) # hcw cutmix
self.labels = tf.placeholder(tf.float32, [None, None]) # hcw cutmix
self.is_training = tf.placeholder(tf.bool, name='is_training') #hcw
self.keep_prob = tf.placeholder(tf.float32, name='keep_prob') #hcw
if model == 'inception_resnet_v2':
with slim.arg_scope(inception_resnet_v2_arg_scope()):
logits, end_points = inception_resnet_v2(self.input, class_num, is_training = self.is_training, dropout_keep_prob = self.keep_prob) #hcw
self.exclude = ['InceptionResnetV2/AuxLogits', 'InceptionResnetV2/Logits']
self.last_layer_name = 'Predictions'
self.path_pretrain = root_pretrain + 'inception_resnet_v2.ckpt'
elif model == 'resnet_v2_50':
with slim.arg_scope(resnet_arg_scope()):
logits, end_points = resnet_v2_50(self.input, class_num, is_training = self.is_training)
self.exclude = ['resnet_v2_50/logits']
self.last_layer_name = 'predictions'
self.path_pretrain = root_pretrain + 'resnet_v2_50.ckpt'
elif model == 'mobilenet_v2':
logits, end_points = mobilenet(self.input, class_num, is_training = self.is_training, depth_multiplier=0.5, finegrain_classification_mode=True)
self.exclude = ['MobilenetV2/Logits']
self.last_layer_name = 'Predictions'
self.path_pretrain = root_pretrain + 'mobilenet_v2_0.5_128.ckpt'
# Wrappers for mobilenet v2 with depth-multipliers. Be noticed that
# 'finegrain_classification_mode' is set to True, which means the embedding
# layer will not be shrinked when given a depth-multiplier < 1.0.
else:
raise ValueError('Error: the model is not available.')
return logits, end_points
def testEndPointsV2(self):
"""Test the end points of a tiny v2 bottleneck network."""
bottleneck = resnet_v2.bottleneck
blocks = [
resnet_utils.Block('block1', bottleneck, [(4, 1, 1), (4, 1, 2)]),
resnet_utils.Block('block2', bottleneck, [(8, 2, 1), (8, 2, 1)])
]
inputs = create_test_input(2, 32, 16, 3)
with slim.arg_scope(resnet_utils.resnet_arg_scope()):
_, end_points = self._resnet_plain(inputs, blocks, scope='tiny')
expected = [
'tiny/block1/unit_1/bottleneck_v2/shortcut',
'tiny/block1/unit_1/bottleneck_v2/conv1',
'tiny/block1/unit_1/bottleneck_v2/conv2',
'tiny/block1/unit_1/bottleneck_v2/conv3',
'tiny/block1/unit_2/bottleneck_v2/conv1',
'tiny/block1/unit_2/bottleneck_v2/conv2',
'tiny/block1/unit_2/bottleneck_v2/conv3',
'tiny/block2/unit_1/bottleneck_v2/shortcut',
'tiny/block2/unit_1/bottleneck_v2/conv1',
'tiny/block2/unit_1/bottleneck_v2/conv2',
'tiny/block2/unit_1/bottleneck_v2/conv3',
'tiny/block2/unit_2/bottleneck_v2/conv1',
'tiny/block2/unit_2/bottleneck_v2/conv2',
'tiny/block2/unit_2/bottleneck_v2/conv3'
]
self.assertItemsEqual(expected, end_points)
def testAtrousFullyConvolutionalValues(self):
"""Verify dense feature extraction with atrous convolution."""
nominal_stride = 32
for output_stride in [4, 8, 16, 32, None]:
with slim.arg_scope(resnet_utils.resnet_arg_scope()):
with tf.Graph().as_default():
with self.test_session() as sess:
tf.set_random_seed(0)
inputs = create_test_input(2, 81, 81, 3)
# Dense feature extraction followed by subsampling.
output, _ = self._resnet_small(
inputs,
None,
is_training=False,
global_pool=False,
output_stride=output_stride)
if output_stride is None:
factor = 1
else:
factor = nominal_stride // output_stride
output = resnet_utils.subsample(output, factor)
# Make the two networks use the same weights.
tf.get_variable_scope().reuse_variables()
# Feature extraction at the nominal network rate.
expected, _ = self._resnet_small(inputs,
None,
is_training=False,
global_pool=False)
sess.run(tf.global_variables_initializer())
self.assertAllClose(output.eval(),
expected.eval(),
atol=1e-4,
rtol=1e-4)
def __init__(self, hands, gestures):
self._image = tf.placeholder(tf.float32, [128, 128, 3])
self._standardized_images = tf.expand_dims(
tf.image.per_image_standardization(self._image), 0)
print self._standardized_images
with slim.arg_scope(resnet_utils.resnet_arg_scope()):
logits_tensor, end_points = resnet_v2_50(self._standardized_images,
gestures, False)
saver = tf.train.Saver()
sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True))
tf.train.start_queue_runners(sess)
self.probabilities_tensor = end_points['predictions']
if hands == "RH":
ckpt = "/s/red/a/nobackup/cwc/tf/hands_demo_rgb/RH_fine/model.ckpt-5590"
else:
ckpt = "/s/red/a/nobackup/cwc/tf/hands_demo_rgb/LH_fine/model.ckpt-40734"
print 'Loading checkpoint %s' % ckpt
saver.restore(sess, ckpt)
self.sess = sess
self.past_probs = None
def resnet(x, is_training, activation, rn_type='50'):
scope = rnu.resnet_arg_scope(activation_fn=activation)
with slim.arg_scope(scope):
if rn_type == '200':
return rn.resnet_v2_200(x, is_training)
return rn.resnet_v2_50(x, is_training)
def resnet_tensorboard():
x_input = tf.placeholder(dtype=tf.float32, shape=(None, image_size, image_size, 3))
arg_scope = resnet_utils.resnet_arg_scope()
with slim.arg_scope(arg_scope):
logits_50, end_points_50 = resnet_v1.resnet_v1_50(x_input,
num_classes=1000,
is_training=False,
global_pool=True,
output_stride=None,
spatial_squeeze=True,
store_non_strided_activations=False,
reuse=False,
scope='resnet_v1_50')
logits_101, end_points_101 = resnet_v1.resnet_v1_101(x_input,
num_classes=1000,
is_training=False,
global_pool=True,
output_stride=None,
spatial_squeeze=True,
store_non_strided_activations=False,
reuse=False,
scope='resnet_v1_101')
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
with tf.Session(config= config) as sess:
sess.run(tf.global_variables_initializer())
sess.run(tf.local_variables_initializer())
summary_writer = tf.summary.FileWriter('/Users/alexwang/data/resnet_summary', graph=sess.graph)
summary_writer.close()
def testEndpointNames(self):
# Like ResnetUtilsTest.testEndPointsV2(), but for the public API.
global_pool = True
num_classes = 10
inputs = create_test_input(2, 224, 224, 3)
with slim.arg_scope(resnet_utils.resnet_arg_scope()):
_, end_points = self._resnet_small(inputs,
num_classes,
global_pool=global_pool,
scope='resnet')
expected = ['resnet/conv1']
for block in range(1, 5):
for unit in range(1, 4 if block < 4 else 3):
for conv in range(1, 4):
expected.append(
'resnet/block%d/unit_%d/bottleneck_v2/conv%d' %
(block, unit, conv))
expected.append('resnet/block%d/unit_%d/bottleneck_v2' %
(block, unit))
expected.append('resnet/block%d/unit_1/bottleneck_v2/shortcut' %
block)
expected.append('resnet/block%d' % block)
expected.extend([
'global_pool', 'resnet/logits', 'resnet/spatial_squeeze',
'predictions'
])
self.assertItemsEqual(end_points.keys(), expected)
def build_model(self):
print('\nBuilding Model')
# Creating placeholders for the question and the answer
self.questions = tf.placeholder(tf.int64, shape=[None, 15], name="question_vector")
self.answers = tf.placeholder(tf.float32, shape=[None, self.most_freq_limit], name="answer_vector")
self.images = tf.placeholder(tf.float32, shape=[None, 448, 448, 3], name="images_matrix")
arg_scope = resnet_arg_scope()
with tf.contrib.slim.arg_scope(arg_scope):
resnet_features, _ = resnet_v2_152(self.images, reuse=tf.AUTO_REUSE)
depth_norm = tf.norm(resnet_features, ord='euclidean', keepdims=True, axis=3) + 1e-8
self.image_features = resnet_features/depth_norm
with tf.variable_scope("text_features") as scope:
if self.reuse:
scope.reuse_variables()
self.word_embeddings = tf.get_variable('word_embeddings',
[self.vocabulary_size,
self.embedding_size],
initializer=tf.contrib.layers.xavier_initializer())
word_vectors = tf.nn.embedding_lookup(self.word_embeddings, self.questions)
len_word = self._len_seq(word_vectors)
embedded_sentence = tf.nn.dropout(tf.nn.tanh(word_vectors, name="embedded_sentence"),
keep_prob=self.dropout_prob)
lstm = tf.nn.rnn_cell.LSTMCell(self.state_size,
initializer=tf.contrib.layers.xavier_initializer())
_, final_state = tf.nn.dynamic_rnn(lstm, embedded_sentence,
sequence_length=len_word,
dtype=tf.float32)
self.text_features = final_state.c
self.attention_features = self.compute_attention(self.image_features,
self.text_features)
with tf.variable_scope("fully_connected") as scope:
if self.reuse:
scope.reuse_variables()
self.fc1 = tf.nn.dropout(tf.nn.relu(self.fc_layer(self.attention_features, 1024, name="fc1")),
keep_prob=self.dropout_prob)
self.fc2 = self.fc_layer(self.fc1, 3000, name="fc2")
self.answer_prob = tf.nn.softmax(self.fc2)
self.loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits_v2(labels=self.answers,
logits=self.fc2))
self.global_step = tf.Variable(0, name='global_step', trainable=False, dtype=tf.int32)
self.inc = tf.assign_add(self.global_step, 1, name='increment')
self.lr = tf.train.exponential_decay(learning_rate=self.init_lr,
global_step=self.global_step,
decay_steps=10000,
decay_rate=0.5,
staircase=True)
self.optimizer = tf.train.AdamOptimizer(self.lr, beta1=0.9, beta2=0.999, name="optim")
def forward():
import random
import cv2
import matplotlib.pyplot as plt
gesture_list = os.listdir(
"/s/red/a/nobackup/cwc/hands/rgb_hands_new_frames/s_01")
gesture_list.sort()
print gesture_list
#lh_list = np.load("/s/red/a/nobackup/cwc/hands/rgb_test/LH.npy")
if FLAGS.hand == "RH":
rh_list = np.load("/s/red/a/nobackup/cwc/hands/rgb_test/LH.npy")
ckpt = "/s/red/a/nobackup/cwc/tf/hands_demo_rgb/RH_fine/model.ckpt-5590"
else:
rh_list = np.load("/s/red/a/nobackup/cwc/hands/rgb_test/RH.npy")
ckpt = "/s/red/a/nobackup/cwc/tf/hands_demo_rgb/LH_fine/model.ckpt-40734"
checkpoint_file = tf.train.latest_checkpoint(log_root)
print(checkpoint_file)
_image = tf.placeholder(tf.float32, [128, 128, 3])
_standardized_images = tf.expand_dims(
tf.image.per_image_standardization(_image), 0)
with slim.arg_scope(resnet_utils.resnet_arg_scope()):
logits, end_points = resnet_v2_50(_standardized_images, 20)
print logits, end_points
variables_to_restore = slim.get_variables_to_restore()
# State the metrics that you want to predict. We get a predictions that is not one_hot_encoded.
predictions = tf.argmax(end_points['predictions'], 1)
probabilities = end_points['predictions']
saver = tf.train.Saver(max_to_keep=0)
sess = tf.Session()
#saver.restore(sess, tf.train.latest_checkpoint(log_root))
saver.restore(sess, ckpt)
for i in range(10):
index = random.randint(0, rh_list.shape[0] - 1)
rh_image = rh_list[index]
rh_image = rh_image[:, :, [2, 1, 0]]
rh_image = cv2.resize(rh_image, (128, 128))
print rh_image.shape
pred_index = sess.run(predictions, feed_dict={_image: rh_image})
pred_gesture = gesture_list[pred_index[0]]
plt.imshow(rh_image)
plt.title(pred_gesture)
plt.show()
def __init__(self,
net_input,
mission,
net_name,
stack,
block1=3,
block2=4,
block3=6,
block4=3,
feature_channels=512,
build_pyramid=True,
build_pyramid_layers=4,
output_stride=None,
is_training=True,
include_root_block=True,
dropout_keep_prob=0.8,
net_head='default',
fusion='sum',
deformable=None,
attention_option=None,
reuse=None):
self.net_input = net_input
self.mission = mission
self.net_name = net_name
self.build_pyramid = build_pyramid
self.build_pyramid_layers = stack
self.is_training = is_training
self.output_stride = output_stride
self.include_root_block = include_root_block
self.dropout_keep_prob = dropout_keep_prob
self.net_head = net_head
self.fusion = fusion
self.deformable = deformable
self.attention_option = attention_option
self.net_arg_scope = resnet_utils.resnet_arg_scope()
self.scope = mission + '/' + net_name
self.reuse = reuse
self.end_points = {}
self.predict_layers = []
self.block_num = [block1, block2, block3, block4]
self.feature_channels = feature_channels
with slim.arg_scope(self.net_arg_scope):
if net_name == 'resnet_v2':
self.resnet_v2_net()
elif net_name == 'resnext_v2':
self.resnext_v2_net()
else:
raise Exception('there is no net called %s' % net_name)
self.resnet_v2()
def testFullyConvolutionalUnknownHeightWidth(self):
batch = 2
height, width = 65, 65
global_pool = False
inputs = create_test_input(batch, None, None, 3)
with slim.arg_scope(resnet_utils.resnet_arg_scope()):
output, _ = self._resnet_small(inputs, None, global_pool=global_pool)
self.assertListEqual(output.get_shape().as_list(),
[batch, None, None, 32])
images = create_test_input(batch, height, width, 3)
with self.test_session() as sess:
sess.run(tf.global_variables_initializer())
output = sess.run(output, {inputs: images.eval()})
self.assertEqual(output.shape, (batch, 3, 3, 32))
def build_model(self, is_training=True, dropout_keep_prob=0.5):
self.inputs = tf.placeholder(real_type(self.FLAGS),
[self.FLAGS.batch_size, 224, 224, 3])
self.targets = tf.placeholder(tf.int32, [self.FLAGS.batch_size])
with slim.arg_scope(resnet_utils.resnet_arg_scope(is_training)):
logits, endpoints = resnet_v1.resnet_v1_101(
self.inputs, self.FLAGS.num_classes)
logits = tf.squeeze(logits)
loss = tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=logits, labels=self.targets)
self.cost = tf.reduce_sum(loss)
self.global_step = tf.contrib.framework.get_or_create_global_step()
self.train_op = tf.train.AdagradOptimizer(0.01).minimize(
loss, global_step=self.global_step)
def testClassificationEndPoints(self):
global_pool = True
num_classes = 10
inputs = create_test_input(2, 224, 224, 3)
with slim.arg_scope(resnet_utils.resnet_arg_scope()):
logits, end_points = self._resnet_small(inputs,
num_classes,
global_pool=global_pool,
scope='resnet')
self.assertTrue(logits.op.name.startswith('resnet/logits'))
self.assertListEqual(logits.get_shape().as_list(),
[2, 1, 1, num_classes])
self.assertTrue('predictions' in end_points)
self.assertListEqual(end_points['predictions'].get_shape().as_list(),
[2, 1, 1, num_classes])
def _atrousValues(self, bottleneck):
"""Verify the values of dense feature extraction by atrous convolution.
Make sure that dense feature extraction by stack_blocks_dense() followed by
subsampling gives identical results to feature extraction at the nominal
network output stride using the simple self._stack_blocks_nondense() above.
Args:
bottleneck: The bottleneck function.
"""
blocks = [
resnet_utils.Block('block1', bottleneck, [(4, 1, 1), (4, 1, 2)]),
resnet_utils.Block('block2', bottleneck, [(8, 2, 1), (8, 2, 2)]),
resnet_utils.Block('block3', bottleneck, [(16, 4, 1), (16, 4, 2)]),
resnet_utils.Block('block4', bottleneck, [(32, 8, 1), (32, 8, 1)])
]
nominal_stride = 8
# Test both odd and even input dimensions.
height = 30
width = 31
with slim.arg_scope(resnet_utils.resnet_arg_scope()):
with slim.arg_scope([slim.batch_norm], is_training=False):
for output_stride in [1, 2, 4, 8, None]:
with tf.Graph().as_default():
with self.test_session() as sess:
tf.set_random_seed(0)
inputs = create_test_input(1, height, width, 3)
# Dense feature extraction followed by subsampling.
output = resnet_utils.stack_blocks_dense(
inputs, blocks, output_stride)
if output_stride is None:
factor = 1
else:
factor = nominal_stride // output_stride
output = resnet_utils.subsample(output, factor)
# Make the two networks use the same weights.
tf.get_variable_scope().reuse_variables()
# Feature extraction at the nominal network rate.
expected = self._stack_blocks_nondense(
inputs, blocks)
sess.run(tf.global_variables_initializer())
output, expected = sess.run([output, expected])
self.assertAllClose(output,
expected,
atol=1e-4,
rtol=1e-4)
def testClassificationShapes(self):
global_pool = True
num_classes = 10
inputs = create_test_input(2, 224, 224, 3)
with slim.arg_scope(resnet_utils.resnet_arg_scope()):
_, end_points = self._resnet_small(inputs, num_classes,
global_pool=global_pool,
scope='resnet')
endpoint_to_shape = {
'resnet/block1': [2, 28, 28, 4],
'resnet/block2': [2, 14, 14, 8],
'resnet/block3': [2, 7, 7, 16],
'resnet/block4': [2, 7, 7, 32]}
for endpoint in endpoint_to_shape:
shape = endpoint_to_shape[endpoint]
self.assertListEqual(end_points[endpoint].get_shape().as_list(), shape)
def testFullyConvolutionalEndpointShapes(self):
global_pool = False
num_classes = 10
inputs = create_test_input(2, 321, 321, 3)
with slim.arg_scope(resnet_utils.resnet_arg_scope()):
_, end_points = self._resnet_small(inputs, num_classes,
global_pool=global_pool,
spatial_squeeze=False,
scope='resnet')
endpoint_to_shape = {
'resnet/block1': [2, 41, 41, 4],
'resnet/block2': [2, 21, 21, 8],
'resnet/block3': [2, 11, 11, 16],
'resnet/block4': [2, 11, 11, 32]}
for endpoint in endpoint_to_shape:
shape = endpoint_to_shape[endpoint]
self.assertListEqual(end_points[endpoint].get_shape().as_list(), shape)
def testUnknownBatchSize(self):
batch = 2
height, width = 65, 65
global_pool = True
num_classes = 10
inputs = create_test_input(None, height, width, 3)
with slim.arg_scope(resnet_utils.resnet_arg_scope()):
logits, _ = self._resnet_small(inputs, num_classes,
global_pool=global_pool,
spatial_squeeze=False,
scope='resnet')
self.assertTrue(logits.op.name.startswith('resnet/logits'))
self.assertListEqual(logits.get_shape().as_list(),
[None, 1, 1, num_classes])
images = create_test_input(batch, height, width, 3)
with self.test_session() as sess:
sess.run(tf.global_variables_initializer())
output = sess.run(logits, {inputs: images.eval()})
self.assertEqual(output.shape, (batch, 1, 1, num_classes))
def testRootlessFullyConvolutionalEndpointShapes(self):
global_pool = False
num_classes = 10
inputs = create_test_input(2, 128, 128, 3)
with slim.arg_scope(resnet_utils.resnet_arg_scope()):
_, end_points = self._resnet_small(inputs,
num_classes,
global_pool=global_pool,
include_root_block=False,
scope='resnet')
endpoint_to_shape = {
'resnet/block1': [2, 64, 64, 4],
'resnet/block2': [2, 32, 32, 8],
'resnet/block3': [2, 16, 16, 16],
'resnet/block4': [2, 16, 16, 32]
}
for endpoint in endpoint_to_shape:
shape = endpoint_to_shape[endpoint]
self.assertListEqual(
end_points[endpoint].get_shape().as_list(), shape)
def tensorflow_resnet_model(self, images):
image_shape = [224, 224, 3]
inputs = images = tf.placeholder(dtype=tf.float32,
shape=[self.config.BATCH_SIZE] +
image_shape)
with tf.contrib.slim.arg_scope(resnet_utils.resnet_arg_scope()):
logits, endPoints = resnet_v1_50(inputs, num_classes=1000)
probs = tf.nn.softmax(endPoints['predictions'])
saver = tf.train.Saver()
sess = tf.Session()
saver.restore(sess, "./resnet_v1_50.ckpt")
img1 = imread('./laska.png', mode='RGB')
img1 = imresize(img1, (224, 224))
prob = sess.run(probs, feed_dict={inputs: [img1]})[0]
print(len(prob))
preds = (np.argsort(prob)[::-1])
for p in preds:
print(class_names[p], prob[p])
include_root_block=True,
spatial_squeeze=True,
reuse=None,
scope='resnet_v1_50'):
blocks = [
resnet_utils.Block('block1', bottleneck,
[(256, 64, 1)] * 2 + [(256, 64, 2)]),
resnet_utils.Block('block2', bottleneck,
[(512, 128, 1)] * 3 + [(512, 128, 2)]),
resnet_utils.Block('block3', bottleneck,
[(1024, 256, 1)] * 5 + [(1024, 256, 2)]),
resnet_utils.Block('block4', bottleneck, [(2048, 512, 1)] * 3)
]
return resnet_v1(inputs,
blocks,
num_classes=num_classes,
is_training=is_training,
global_pool=global_pool,
output_stride=output_stride,
include_root_block=include_root_block,
spatial_squeeze=spatial_squeeze,
reuse=reuse,
scope=scope)
if __name__ == '__main__':
input = tf.placeholder(tf.float32, shape=(None, 224, 224, 3), name='input')
with slim.arg_scope(resnet_utils.resnet_arg_scope()) as sc:
logits = resnet_v1_50(input)
def train():
tf.logging.set_verbosity(
tf.logging.INFO) # Set the verbosity to INFO level
initial_learning_rate = 0.0002
learning_rate_decay_factor = 0.7
decay_steps = 40000
checkpoint_file = tf.train.latest_checkpoint(log_root)
print(checkpoint_file)
images, one_hot_labels = get_input(20, FLAGS.batch_size, hand=FLAGS.hand)
with slim.arg_scope(resnet_utils.resnet_arg_scope()):
logits, end_points = resnet_v2_50(images, 20)
print logits, end_points
variables_to_restore = slim.get_variables_to_restore()
loss = tf.losses.softmax_cross_entropy(onehot_labels=one_hot_labels,
logits=logits)
total_loss = tf.losses.get_total_loss(
) # obtain the regularization losses as well
# Create the global step for monitoring the learning_rate and training.
global_step_1 = tf.contrib.framework.get_or_create_global_step()
# Define your exponentially decaying learning rate
lr = tf.train.exponential_decay(learning_rate=initial_learning_rate,
global_step=global_step_1,
decay_steps=decay_steps,
decay_rate=learning_rate_decay_factor,
staircase=True)
# Now we can define the optimizer that takes on the learning rate
optimizer = tf.train.AdamOptimizer(learning_rate=lr)
#variables_to_train = [v for v in variables_to_restore if 'resnet_v2_50/logits' in v.name]
#print variables_to_train
#raw_input()
# Create the train_op.
train_op = slim.learning.create_train_op(
total_loss,
optimizer,
) # variables_to_train = variables_to_train)
# State the metrics that you want to predict. We get a predictions that is not one_hot_encoded.
predictions = tf.argmax(end_points['predictions'], 1)
probabilities = end_points['predictions']
labels = tf.argmax(one_hot_labels, 1)
precision = tf.reduce_mean(tf.to_float(tf.equal(predictions, labels)))
summary_writer = tf.summary.FileWriter(train_dir)
summary_op = tf.summary.merge([
tf.summary.scalar('costs/cost', total_loss),
tf.summary.scalar('Precision', precision),
tf.summary.scalar('learning_rate', lr)
])
saver = tf.train.Saver(max_to_keep=0)
sess = tf.Session()
saver.restore(sess, tf.train.latest_checkpoint(log_root))
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
var = [
v for v in tf.model_variables()
if "50/conv1" in v.name or "logits" in v.name
]
var_list = sess.run(var)
for v1, v in zip(var, var_list):
print v1.name, v.shape, np.min(v), np.max(v), np.mean(v), np.std(v)
saver.save(sess,
os.path.join(log_root, "model.ckpt"),
global_step=global_step_1)
checkpoint_time = time.time()
while True:
_, summary, g_step, loss, accuracy, l_rate = sess.run(
[train_op, summary_op, global_step_1, total_loss, precision, lr])
if g_step % 10 == 0:
tf.logging.info(
'Global_step_1: %d, loss: %f, precision: %.2f, lr: %f' %
(g_step, loss, accuracy, l_rate))
if time.time() - checkpoint_time > save_checkpoint_secs:
saver.save(sess,
os.path.join(log_root, "model.ckpt"),
global_step=global_step_1)
checkpoint_time = time.time()
if g_step % save_summaries_steps == 0:
summary_writer.add_summary(summary, g_step)
summary_writer.flush()
def evaluate():
"""Eval loop."""
images, one_hot_labels = get_input(20, FLAGS.batch_size, FLAGS.mode,
FLAGS.hand)
gesture_list = os.listdir(
"/s/red/a/nobackup/cwc/hands/rgb_hands_new_frames/s_01")
gesture_list.sort()
print gesture_list
with slim.arg_scope(resnet_utils.resnet_arg_scope()):
logits_tensor, end_points = resnet_v2_50(images, 20, False)
saver = tf.train.Saver()
summary_writer = tf.summary.FileWriter(eval_dir)
sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True))
tf.train.start_queue_runners(sess)
best_precision = 0.0
predictions_tensor = tf.argmax(end_points['predictions'], 1)
probabilities_tensor = end_points['predictions']
labels = tf.argmax(one_hot_labels, 1)
#precision = tf.reduce_mean(tf.to_float(tf.equal(predictions_tensor, labels)))
checkpoint_list = os.listdir(log_root)
checkpoint_list = [
c.replace(".index", "") for c in checkpoint_list if ".index" in c
]
print checkpoint_list
precision_list = []
#while True:
for c in checkpoint_list:
try:
ckpt_state = tf.train.get_checkpoint_state(log_root)
except tf.errors.OutOfRangeError as e:
tf.logging.error('Cannot restore checkpoint: %s', e)
continue
if not (ckpt_state and ckpt_state.model_checkpoint_path):
tf.logging.info('No model to eval yet at %s', log_root)
continue
ckpt_state.model_checkpoint_path = os.path.join(log_root, c)
tf.logging.info('Loading checkpoint %s',
ckpt_state.model_checkpoint_path)
saver.restore(sess, ckpt_state.model_checkpoint_path)
tf.logging.info("Checkpoint restoration done")
train_step = int(ckpt_state.model_checkpoint_path.split("-")[-1])
total_prediction, correct_prediction = 0, 0
gt_list = []
pred_list = []
for sample in range(212):
(predictions,
truth) = sess.run([probabilities_tensor, one_hot_labels])
gt_list.append(truth)
pred_list.append(predictions)
truth = np.argmax(truth, axis=1)
predictions = np.argmax(predictions, axis=1)
correct_prediction += np.sum(truth == predictions)
total_prediction += predictions.shape[0]
if sample % 40 == 0:
print sample, correct_prediction, total_prediction
gt = np.squeeze(np.vstack(gt_list))
pred = np.squeeze(np.vstack(pred_list))
precision = np.mean(np.argmax(gt, axis=1) == np.argmax(pred, axis=1))
precision_list.append(precision)
best_precision = max(precision, best_precision)
print gt.shape, pred.shape, precision, best_precision
gt = np.argmax(gt, axis=1)
pred = np.argmax(pred, axis=1)
'''cm = np.float(confusion_matrix(gt,pred))
cm /= np.sum(cm,axis=0)
plt.imshow(cm,interpolation='nearest')
plt.xticks(range(20),gesture_list,rotation=90)
plt.yticks(range(20),gesture_list)
plt.show()
precision_summ = tf.Summary()
precision_summ.value.add(tag='Precision', simple_value=precision)
summary_writer.add_summary(precision_summ, train_step)
best_precision_summ = tf.Summary()
best_precision_summ.value.add(tag='Best Precision', simple_value=best_precision)
summary_writer.add_summary(best_precision_summ, train_step)'''
tf.logging.info('precision: %.3f, best precision: %.3f' %
(precision, best_precision))
summary_writer.flush()
#time.sleep(60)
print "Max Precision : ", np.max(
precision_list), " Ckpt: ", checkpoint_list[np.argmax(precision_list)]
def build_model(self):
self.is_training = tf.placeholder(tf.bool, [])
self.step = tf.get_variable("global_step", [],
initializer=tf.constant_initializer(0.0),
trainable=False)
lr = tf.train.exponential_decay(learning_rate=1e-2,
global_step=self.step,
decay_steps=10000,
decay_rate=0.1,
staircase=True)
optimizer = tf.train.GradientDescentOptimizer(learning_rate=lr)
# opt_init = tf.train.GradientDescentOptimizer(learning_rate=lr)
labels_all = []
tower_grads = []
eval_logits = []
with tf.variable_scope(tf.get_variable_scope()):
for i in range(num_gpus):
print('\n num gpu:{}\n'.format(i))
with tf.device('/gpu:%d' % i), tf.name_scope(
'%s_%d' % ("classification", i)) as scope:
imgs_batch, label_batch = resnet_wildcat.data_load(args)
labels_all.append(label_batch)
with slim.arg_scope(resnet_utils.resnet_arg_scope()):
logits, end_points, net_conv5 = resnet_v1_101.resnet_v1_101(
imgs_batch,
num_classes=args.class_num,
is_training=args.is_training,
global_pool=True,
output_stride=None,
spatial_squeeze=True,
store_non_strided_activations=False,
reuse=None,
scope='resnet_v1_101')
tf.losses.sigmoid_cross_entropy(label_batch, logits)
# update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS, scope)
# updates_op = tf.group(*update_ops)
# with tf.control_dependencies([updates_op]):
# cross_entropy = tf.identity(cross_entropy, name='train_op')
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS,
scope)
updates_op = tf.group(*update_ops)
with tf.control_dependencies([updates_op]):
losses = tf.get_collection(tf.GraphKeys.LOSSES, scope)
total_loss = tf.add_n(losses, name='total_loss')
# if update_ops:
# updates = tf.group(*update_ops)
# cross_entropy = control_flow_ops.with_dependencies([updates], cross_entropy)
# reuse var
tf.get_variable_scope().reuse_variables()
# just an assertion!
assert tf.get_variable_scope().reuse == True
# grad compute
# if args.is_training:
grads = optimizer.compute_gradients(total_loss)
# important!!! logits/biases is None but not tensor, no gradient in it
new_grads = []
for gv in grads:
if gv[0] is not None:
new_grads.append(gv)
tower_grads.append(new_grads)
eval_logits.append(tf.nn.sigmoid(logits))
# We must calculate the mean of each gradient
# if training:
grads = multi_gpu.average_gradients(tower_grads)
# Apply the gradients to adjust the shared variables.
apply_gradient_op = optimizer.apply_gradients(grads,
global_step=self.step)
# Group all updates to into a single train op.
self.train_op = tf.group(apply_gradient_op)
self.prediction = tf.concat(eval_logits, axis=0)
self.cross_entropy = total_loss
self.label_batch = tf.concat(labels_all, axis=0)
merged_summary_op = tf.summary.merge_all()
# load weights// frist to initializer all vars
init = tf.global_variables_initializer(
) # tf.variables_initializer(var_list=initvars)
self.sess.run(init)
all_variables = tf.global_variables()
# for var in all_variables:
# print(var)
# init vars
load_vars = [v for v in all_variables if 'step' not in v.name]
self.saver = tf.train.Saver(var_list=load_vars)
frist_load_model = False
if frist_load_model:
resnet101_model_path = '/home/liuweiwei02/Projects/resnet_v1_101.ckpt'
exclude = ['resnet_v1_101/logits']
resnet_vars = slim.get_variables_to_restore(
include=['resnet_v1_101'], exclude=exclude)
init_fn = slim.assign_from_checkpoint_fn(resnet101_model_path,
resnet_vars)
init_fn(sess)
print('resnet_model load done. \n')
with slim.arg_scope(resnet_v1.resnet_arg_scope()):
net, end_points = resnet_v1.resnet_v1_101(inputs,
21,
is_training=False,
global_pool=False,
output_stride=16)
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import tensorflow as tf
import resnet_utils
resnet_arg_scope = resnet_utils.resnet_arg_scope(weight_decay=0.0)
slim = tf.contrib.slim
@slim.add_arg_scope
def bottleneck(inputs,
depth,
depth_bottleneck,
stride,
rate=1,
noise_fn=None,
outputs_collections=None,
scope=None):
"""Bottleneck residual unit variant with BN after convolutions.
This is the original residual unit proposed in [1]. See Fig. 1(a) of [2] for
tf.FixedLenFeature([], tf.int64),
"image":
tf.FixedLenFeature([], tf.string)
})
img = tf.decode_raw(features["image"], tf.uint8)
img = tf.reshape(img, [image_pixels, image_pixels, 3])
img = tf.cast(img, tf.float32)
label = tf.cast(features["label"], tf.int32)
return img, label
images = tf.placeholder(tf.float32, [None, image_pixels, image_pixels, 3],
name="input/x_input")
labels = tf.placeholder(tf.int64, [None], name="input/y_input")
with slim.arg_scope(resnet_arg_scope()):
logits, end_points = resnet_v2_152(images,
num_classes=classes,
is_training=True)
exclude = ["resnet_v2_152/logits"]
variables_to_restore = slim.get_variables_to_restore(exclude=exclude)
learning_rate = tf.Variable(initial_value=0.00003,
trainable=False,
name="learning_rate",
dtype=tf.float32)
update_learning_rate = tf.assign(learning_rate, learning_rate * 0.8)
one_hot_labels = slim.one_hot_encoding(labels, classes)
loss = tf.losses.softmax_cross_entropy(onehot_labels=one_hot_labels,
logits=logits)
def testStridingLastUnitVsSubsampleBlockEnd(self):
"""Compares subsampling at the block's last unit or block's end.
Makes sure that the final output is the same when we use a stride at the
last unit of a block vs. we subsample activations at the end of a block.
"""
block = resnet_v1.resnet_v1_block
blocks = [
block('block1', base_depth=1, num_units=2, stride=2),
block('block2', base_depth=2, num_units=2, stride=2),
block('block3', base_depth=4, num_units=2, stride=2),
block('block4', base_depth=8, num_units=2, stride=1),
]
# Test both odd and even input dimensions.
height = 30
width = 31
with slim.arg_scope(resnet_utils.resnet_arg_scope()):
with slim.arg_scope([slim.batch_norm], is_training=False):
for output_stride in [1, 2, 4, 8, None]:
with tf.Graph().as_default():
with self.test_session() as sess:
tf.set_random_seed(0)
inputs = create_test_input(1, height, width, 3)
# Subsampling at the last unit of the block.
output = resnet_utils.stack_blocks_dense(
inputs, blocks, output_stride,
store_non_strided_activations=False,
outputs_collections='output')
output_end_points = slim.utils.convert_collection_to_dict(
'output')
# Make the two networks use the same weights.
tf.get_variable_scope().reuse_variables()
# Subsample activations at the end of the blocks.
expected = resnet_utils.stack_blocks_dense(
inputs, blocks, output_stride,
store_non_strided_activations=True,
outputs_collections='expected')
expected_end_points = slim.utils.convert_collection_to_dict(
'expected')
sess.run(tf.global_variables_initializer())
# Make sure that the final output is the same.
output, expected = sess.run([output, expected])
self.assertAllClose(output, expected, atol=1e-4, rtol=1e-4)
# Make sure that intermediate block activations in
# output_end_points are subsampled versions of the corresponding
# ones in expected_end_points.
for i, block in enumerate(blocks[:-1:]):
output = output_end_points[block.scope]
expected = expected_end_points[block.scope]
atrous_activated = (output_stride is not None and
2 ** i >= output_stride)
if not atrous_activated:
expected = resnet_utils.subsample(expected, 2)
output, expected = sess.run([output, expected])
self.assertAllClose(output, expected, atol=1e-4, rtol=1e-4)
def build_model(self):
print('\nBuilding Model')
# Creating placeholders for the question and the answer
self.questions = tf.placeholder(tf.int64,
shape=[None, 15],
name="question_vector")
self.answers = tf.placeholder(tf.float32,
shape=[None, self.most_freq_limit],
name="answer_vector")
self.images = tf.placeholder(tf.float32,
shape=[None, 448, 448, 3],
name="images_matrix")
arg_scope = resnet_arg_scope()
with tf.contrib.slim.arg_scope(arg_scope):
resnet_features, _ = resnet_v2_152(self.images,
reuse=tf.AUTO_REUSE)
depth_norm = tf.norm(
resnet_features, ord='euclidean', keepdims=True, axis=3) + 1e-8
self.image_features = resnet_features / depth_norm
with tf.variable_scope("text_features") as scope:
if self.reuse:
scope.reuse_variables()
self.word_embeddings = tf.get_variable(
'word_embeddings', [self.vocabulary_size, self.embedding_size],
initializer=tf.contrib.layers.xavier_initializer())
word_vectors = tf.nn.embedding_lookup(self.word_embeddings,
self.questions)
len_word = self._len_seq(word_vectors)
embedded_sentence = tf.nn.dropout(tf.nn.tanh(
word_vectors, name="embedded_sentence"),
keep_prob=self.dropout_prob)
lstm = tf.nn.rnn_cell.LSTMCell(
self.state_size,
initializer=tf.contrib.layers.xavier_initializer())
_, final_state = tf.nn.dynamic_rnn(lstm,
embedded_sentence,
sequence_length=len_word,
dtype=tf.float32)
self.text_features = final_state.c
self.attention_features = self.compute_attention(
self.image_features, self.text_features)
with tf.variable_scope("fully_connected") as scope:
if self.reuse:
scope.reuse_variables()
self.fc1 = tf.nn.dropout(tf.nn.relu(
self.fc_layer(self.attention_features, 1024, name="fc1")),
keep_prob=self.dropout_prob)
self.fc2 = self.fc_layer(self.fc1, 3000, name="fc2")
self.answer_prob = tf.nn.softmax(self.fc2)
self.loss = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits_v2(labels=self.answers,
logits=self.fc2))
self.global_step = tf.Variable(0,
name='global_step',
trainable=False,
dtype=tf.int32)
self.inc = tf.assign_add(self.global_step, 1, name='increment')
self.lr = tf.train.exponential_decay(learning_rate=self.init_lr,
global_step=self.global_step,
decay_steps=10000,
decay_rate=0.5,
staircase=True)
self.optimizer = tf.train.AdamOptimizer(self.lr,
beta1=0.9,
beta2=0.999,
name="optim")
def _resnet_v2(self, is_training):
with slim.arg_scope(resnet_arg_scope()):
pool, _ = resnet_v2_101(self._image, is_training=is_training)
return pool
