def conv2d(input_data,
kernel_height,
kernel_width,
in_channels,
out_channels,
strides=[1, 1, 1, 1],
padding='SAME',
regularization=False,
kernel_summary=False,
name=None):
if regularization is not True:
kernel = ut._variable_with_weight_decay(
'kernels',
[kernel_height, kernel_width, in_channels, out_channels],
tf.truncated_normal_initializer(stddev=5e-2), 0.0)
else:
kernel = ut._variable_with_weight_decay(
'kernels',
[kernel_height, kernel_width, in_channels, out_channels],
tf.truncated_normal_initializer(stddev=5e-2), 0.001)
biases = ut._variable_on_gpu('biases', [out_channels],
tf.constant_initializer(0.0))
conv = tf.nn.conv2d(input_data, kernel, strides, padding)
conv = tf.nn.bias_add(conv, biases)
conv = tf.nn.relu(conv, name=name)
ut._activation_summary(conv)
if kernel_summary is True:
ut._kernel_summary(kernel, name + '/kernel', out_channels,
kernel_width, kernel_height)
return conv
def fc(input_data, in_channels, out_channels, regularization=True, name=None):
if regularization is True:
weights = ut._variable_with_weight_decay(
'weights', [in_channels, out_channels],
tf.truncated_normal_initializer(stddev=0.05), 0.001)
else:
weights = ut._variable_with_weight_decay(
'weights', [in_channels, out_channels],
tf.truncated_normal_initializer(stddev=0.05), 0.0)
biases = ut._variable_on_gpu('biases', [out_channels],
tf.constant_initializer(0.1))
fc = tf.nn.relu(tf.matmul(input_data, weights) + biases, name=name)
ut._activation_summary(fc)
return fc
def dense_layer(feed,
input_dim,
output_dim,
dropout=False,
keep_prob=None,
batch_norm=False,
weight_decay=None):
weights = _variable_with_weight_decay('weights',
shape=[input_dim, output_dim],
stddev=0.04,
wd=weight_decay)
biases = _variable_on_cpu('biases', [output_dim],
tf.constant_initializer(0.1))
intermediate = tf.matmul(feed, weights)
if batch_norm:
mean, variance = tf.nn.moments(intermediate, axes=[0])
epsilon = 1e-5
gamma = _variable_on_cpu('gammas', [output_dim],
tf.constant_initializer(1.0))
pre_activation = tf.nn.batch_normalization(intermediate, mean,
variance, biases, gamma,
epsilon)
else:
pre_activation = intermediate + biases
if dropout:
pre_activation = tf.nn.dropout(pre_activation,
keep_prob=keep_prob,
name="dropout")
after_activation = tf.nn.relu(pre_activation, name='activated_out')
_activation_summary(after_activation)
return after_activation
def batch_normalized_conv_layer(state_below, scope_name, n_inputs, n_outputs, filter_shape, stddev, wd, eps=.00001, test=False):
"""
Convolutional layer with batch normalization
"""
with tf.variable_scope(scope_name) as scope:
kernel = _variable_with_weight_decay(
"weights", shape=[filter_shape[0], filter_shape[1], n_inputs, n_outputs],
stddev=stddev, wd=wd
)
conv = tf.nn.conv2d(state_below, kernel, [1, 1, 1, 1], padding='SAME')
# get moments
conv_mean, conv_variance = tf.nn.moments(conv, [0, 1, 2])
# get mean and variance variables
mean = _variable_on_cpu("bn_mean", [n_outputs], tf.constant_initializer(0.0), False)
variance = _variable_on_cpu("bn_variance", [n_outputs], tf.constant_initializer(1.0), False)
# assign the moments
if not test:
assign_mean = mean.assign(conv_mean)
assign_variance = variance.assign(conv_variance)
conv_bn = tf.mul((conv - conv_mean), tf.rsqrt(conv_variance + eps), name=scope.name+"_bn")
else:
conv_bn = tf.mul((conv - mean), tf.rsqrt(variance + eps), name=scope.name+"_bn")
beta = _variable_on_cpu("beta", [n_outputs], tf.constant_initializer(0.0))
gamma = _variable_on_cpu("gamma", [n_outputs], tf.constant_initializer(1.0))
bn = tf.add(tf.mul(conv_bn, gamma), beta)
# output = tf.nn.relu(bn, name=scope.name)
output = randomized_relu(bn, .1, name=scope.name, is_training=(not test))
if not test:
output = control_flow_ops.with_dependencies(dependencies=[assign_mean, assign_variance], output_tensor=output)
_activation_summary(output)
return output
def conv2d_stack(feed,
kernel_list,
stride_list,
padding_list,
batch_norm=False):
if not ((len(kernel_list) == len(stride_list)) and
(len(stride_list) == len(padding_list))):
return
inputs = []
inputs.append(feed)
for i in range(len(kernel_list)):
with tf.variable_scope('conv%d' % (i + 1)) as scope:
kernel = _variable_with_weight_decay('weights',
shape=kernel_list[i],
stddev=5e-2,
wd=None)
conv = conv2d(inputs[-1],
kernel,
stride_list[i],
padding=padding_list[i])
biases = _variable_on_cpu('biases', kernel_list[i][-1],
tf.constant_initializer(0.0))
if batch_norm:
mean, variance = tf.nn.moments(conv, axes=[0])
epsilon = 1e-5
gamma = _variable_on_cpu('gammas', kernel_list[i][-1],
tf.constant_initializer(1.0))
pre_activation = tf.nn.batch_normalization(
conv, mean, variance, biases, gamma, epsilon)
else:
pre_activation = tf.nn.bias_add(conv, biases)
after_activation = tf.nn.relu(pre_activation, name='activated_out')
_activation_summary(after_activation)
inputs.append(after_activation)
return inputs[-1]
def batch_normalized_linear_layer(state_below, scope_name, n_inputs, n_outputs, stddev, wd, eps=.00001, test=False):
"""
A linear layer with batch normalization
"""
with tf.variable_scope(scope_name) as scope:
weight = _variable_with_weight_decay(
"weights", shape=[n_inputs, n_outputs],
stddev=stddev, wd=wd
)
act = tf.matmul(state_below, weight)
# get moments
act_mean, act_variance = tf.nn.moments(act, [0])
# get mean and variance variables
mean = _variable_on_cpu('bn_mean', [n_outputs], tf.constant_initializer(0.0), trainable=False)
variance = _variable_on_cpu('bn_variance', [n_outputs], tf.constant_initializer(1.0), trainable=False)
# assign the moments
if not test:
assign_mean = mean.assign(act_mean)
assign_variance = variance.assign(act_variance)
act_bn = tf.mul((act - act_mean), tf.rsqrt(act_variance + eps), name=scope.name+"_bn")
else:
act_bn = tf.mul((act - mean), tf.rsqrt(variance + eps), name=scope.name+"_bn")
beta = _variable_on_cpu("beta", [n_outputs], tf.constant_initializer(0.0))
gamma = _variable_on_cpu("gamma", [n_outputs], tf.constant_initializer(1.0))
bn = tf.add(tf.mul(act_bn, gamma), beta)
# output = tf.nn.relu(bn, name=scope.name)
output = randomized_relu(bn, .1, name=scope.name, is_training=(not test))
if not test:
output = control_flow_ops.with_dependencies(dependencies=[assign_mean, assign_variance], output_tensor=output)
_activation_summary(output)
return output
def linear_layer(state_below,
scope_name,
n_inputs,
n_outputs,
stddev,
wd,
use_nonlinearity=True):
"""
Standard linear neural network layer
"""
with tf.variable_scope(scope_name) as scope:
weights = _variable_with_weight_decay('weights', [n_inputs, n_outputs],
stddev=stddev,
wd=wd)
biases = _variable_on_cpu('biases', [n_outputs],
tf.constant_initializer(0.0))
activation = tf.nn.xw_plus_b(state_below,
weights,
biases,
name="activation")
if use_nonlinearity:
output = tf.nn.relu(activation, name=scope.name)
else:
output = activation
_activation_summary(output)
return output
def batch_normalized_conv_layer(state_below,
scope_name,
n_inputs,
n_outputs,
filter_shape,
stddev,
wd,
eps=.00001,
test=False):
"""
Convolutional layer with batch normalization
"""
with tf.variable_scope(scope_name) as scope:
kernel = _variable_with_weight_decay(
"weights",
shape=[filter_shape[0], filter_shape[1], n_inputs, n_outputs],
stddev=stddev,
wd=wd)
conv = tf.nn.conv2d(state_below, kernel, [1, 1, 1, 1], padding='SAME')
# get moments
conv_mean, conv_variance = tf.nn.moments(conv, [0, 1, 2])
# get mean and variance variables
mean = _variable_on_cpu("bn_mean", [n_outputs],
tf.constant_initializer(0.0), False)
variance = _variable_on_cpu("bn_variance", [n_outputs],
tf.constant_initializer(1.0), False)
# assign the moments
if not test:
assign_mean = mean.assign(conv_mean)
assign_variance = variance.assign(conv_variance)
conv_bn = tf.mul((conv - conv_mean),
tf.rsqrt(conv_variance + eps),
name=scope.name + "_bn")
else:
conv_bn = tf.mul((conv - mean),
tf.rsqrt(variance + eps),
name=scope.name + "_bn")
beta = _variable_on_cpu("beta", [n_outputs],
tf.constant_initializer(0.0))
gamma = _variable_on_cpu("gamma", [n_outputs],
tf.constant_initializer(1.0))
bn = tf.add(tf.mul(conv_bn, gamma), beta)
# output = tf.nn.relu(bn, name=scope.name)
output = randomized_relu(bn,
.1,
name=scope.name,
is_training=(not test))
if not test:
output = control_flow_ops.with_dependencies(
dependencies=[assign_mean, assign_variance],
output_tensor=output)
_activation_summary(output)
return output
def batch_normalized_linear_layer(state_below,
scope_name,
n_inputs,
n_outputs,
stddev,
wd,
eps=.00001,
test=False):
"""
A linear layer with batch normalization
"""
with tf.variable_scope(scope_name) as scope:
weight = _variable_with_weight_decay("weights",
shape=[n_inputs, n_outputs],
stddev=stddev,
wd=wd)
act = tf.matmul(state_below, weight)
# get moments
act_mean, act_variance = tf.nn.moments(act, [0])
# get mean and variance variables
mean = _variable_on_cpu('bn_mean', [n_outputs],
tf.constant_initializer(0.0),
trainable=False)
variance = _variable_on_cpu('bn_variance', [n_outputs],
tf.constant_initializer(1.0),
trainable=False)
# assign the moments
if not test:
assign_mean = mean.assign(act_mean)
assign_variance = variance.assign(act_variance)
act_bn = tf.mul((act - act_mean),
tf.rsqrt(act_variance + eps),
name=scope.name + "_bn")
else:
act_bn = tf.mul((act - mean),
tf.rsqrt(variance + eps),
name=scope.name + "_bn")
beta = _variable_on_cpu("beta", [n_outputs],
tf.constant_initializer(0.0))
gamma = _variable_on_cpu("gamma", [n_outputs],
tf.constant_initializer(1.0))
bn = tf.add(tf.mul(act_bn, gamma), beta)
# output = tf.nn.relu(bn, name=scope.name)
output = randomized_relu(bn,
.1,
name=scope.name,
is_training=(not test))
if not test:
output = control_flow_ops.with_dependencies(
dependencies=[assign_mean, assign_variance],
output_tensor=output)
_activation_summary(output)
return output
def linear_layer(state_below, scope_name, n_inputs, n_outputs, stddev, wd):
"""
Standard linear neural network layer
"""
with tf.variable_scope(scope_name) as scope:
weights = _variable_with_weight_decay(
'weights', [n_inputs, n_outputs],
stddev=stddev, wd=wd
)
biases = _variable_on_cpu(
'biases', [n_outputs], tf.constant_initializer(0.0)
)
output = tf.nn.xw_plus_b(state_below, weights, biases, name=scope.name)
_activation_summary(output)
return output
def conv_layer(state_below, scope_name, n_inputs, n_outputs, filter_shape, stddev, wd):
"""
A Standard convolutional layer
"""
with tf.variable_scope(scope_name) as scope:
kernel = _variable_with_weight_decay(
"weights", shape=[filter_shape[0], filter_shape[1], n_inputs, n_outputs],
wd=wd
)
conv = tf.nn.conv2d(state_below, kernel, [1, 1, 1, 1], padding='SAME')
biases = _variable_on_cpu("biases", [n_outputs], tf.constant_initializer(0.0))
bias = tf.add(conv, biases)
output = tf.nn.relu(bias, name=scope.name)
_activation_summary(output)
return output
def conv_layer(state_below, scope_name, n_inputs, n_outputs, filter_shape,
stddev, wd):
"""
A Standard convolutional layer
"""
with tf.variable_scope(scope_name) as scope:
kernel = _variable_with_weight_decay(
"weights",
shape=[filter_shape[0], filter_shape[1], n_inputs, n_outputs],
wd=wd)
conv = tf.nn.conv2d(state_below, kernel, [1, 1, 1, 1], padding='SAME')
biases = _variable_on_cpu("biases", [n_outputs],
tf.constant_initializer(0.0))
bias = tf.add(conv, biases)
output = tf.nn.relu(bias, name=scope.name)
_activation_summary(output)
return output
def conv_layer_with_bn(inputT, shape, train_phase, activation=True, name=None):
in_channel = shape[2]
out_channel = shape[3]
k_size = shape[0]
with tf.variable_scope(name) as scope:
kernel = _variable_with_weight_decay(
'ort_weights',
shape=shape,
initializer=orthogonal_initializer(),
wd=None)
conv = tf.nn.conv2d(inputT, kernel, [1, 1, 1, 1], padding='SAME')
biases = _variable_on_cpu('biases', [out_channel],
tf.constant_initializer(0.0))
bias = tf.nn.bias_add(conv, biases)
if activation is True:
conv_out = tf.nn.relu(
batch_norm_layer(bias, train_phase, scope.name))
else:
conv_out = batch_norm_layer(bias, train_phase, scope.name)
return conv_out
def run_testing():
with tf.Graph().as_default():
global_step = tf.get_variable('global_step', [],
initializer=tf.constant_initializer(0),
trainable=False)
with tf.variable_scope('var_name') as var_scope:
weights = {
'wc1':
_variable_with_weight_decay('wc1', [3, 3, 3, 3, 64], 0.005),
'wc2':
_variable_with_weight_decay('wc2', [3, 3, 3, 64, 128], 0.005),
'wc3a':
_variable_with_weight_decay('wc3a', [3, 3, 3, 128, 256],
0.005),
'wc3b':
_variable_with_weight_decay('wc3b', [3, 3, 3, 256, 256],
0.005),
'wc4a':
_variable_with_weight_decay('wc4a', [3, 3, 3, 256, 512],
0.005),
'wc4b':
_variable_with_weight_decay('wc4b', [3, 3, 3, 512, 512],
0.005),
'wc5a':
_variable_with_weight_decay('wc5a', [3, 3, 3, 512, 512],
0.005),
'wc5b':
_variable_with_weight_decay('wc5b', [3, 3, 3, 512, 512],
0.005),
#'wd1': _variable_with_weight_decay('wd1', [8192, 4096], 0.005),
#'wd2': _variable_with_weight_decay('wd2', [4096, 4096], 0.005),
#'out': _variable_with_weight_decay('wout', [4096, c3d_model.NUM_CLASSES], 0.005)
}
biases = {
'bc1': _variable_with_weight_decay('bc1', [64], 0.000),
'bc2': _variable_with_weight_decay('bc2', [128], 0.000),
'bc3a': _variable_with_weight_decay('bc3a', [256], 0.000),
'bc3b': _variable_with_weight_decay('bc3b', [256], 0.000),
'bc4a': _variable_with_weight_decay('bc4a', [512], 0.000),
'bc4b': _variable_with_weight_decay('bc4b', [512], 0.000),
'bc5a': _variable_with_weight_decay('bc5a', [512], 0.000),
'bc5b': _variable_with_weight_decay('bc5b', [512], 0.000),
#'bd1': _variable_with_weight_decay('bd1', [4096], 0.000),
#'bd2': _variable_with_weight_decay('bd2', [4096], 0.000),
#'out': _variable_with_weight_decay('bout', [c3d_model.NUM_CLASSES], 0.000),
}
fcn_weights = {
'wconv6':
_variable_with_weight_decay('conv6', [1, 4, 4, 512, 512],
0.005),
'wup6':
_variable_with_weight_decay('up6', [2, 1, 1, 4096, 512],
0.005),
'wup7':
_variable_with_weight_decay('up7', [2, 1, 1, 4096, 4096],
0.005),
'wup8':
_variable_with_weight_decay(
'up8', [2, 1, 1, fcn_model.NUM_CLASSES, 4096], 0.005),
}
fcn_biases = {
'bconv6':
_variable_with_weight_decay('bconv6', [512], 0.000),
'bup6':
_variable_with_weight_decay('bup6', [4096], 0.000),
'bup7':
_variable_with_weight_decay('bup7', [4096], 0.000),
'bup8':
_variable_with_weight_decay('bup8', [fcn_model.NUM_CLASSES],
0.000),
}
with tf.name_scope('inputs'):
images_placeholder, labels_placeholder, keep_pro = placeholder_inputs(
FLAGS.batch_size)
feature_map = c3d_model.inference_c3d(images_placeholder, keep_pro,
FLAGS.batch_size, weights,
biases)
logit = fcn_model.inference_fcn5(feature_map, keep_pro,
FLAGS.batch_size, fcn_weights,
fcn_biases)
loss = fcn_model_loss(logit, labels_placeholder, FLAGS.batch_size)
accuracy = tower_acc(logit, labels_placeholder, FLAGS.batch_size)
predictions = tf.nn.top_k(logit, 1)
# Create a saver for writing training checkpoints.
new_saver = tf.train.Saver(weights.values() + biases.values() +
fcn_weights.values() + fcn_biases.values())
init = tf.global_variables_initializer()
# Create a session for running Ops on the Graph.
sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True))
sess.run(init)
ckpt = tf.train.get_checkpoint_state(pre_model_save_dir)
if ckpt and ckpt.model_checkpoint_path:
print "loading checkpoint,waiting......"
new_saver.restore(sess, ckpt.model_checkpoint_path)
print "load complete!"
if FLAGS.output_to_file:
# all output will be stored in 'output.txt'
print('outputs will be stored in test.txt')
sys.stdout = open('test.txt', 'a', 1)
predict_list = []
label_list = []
for i in xrange(3358):
start_time = time.time()
test_images, test_labels, _, _, _, _ = input_test_data.read_clip_and_label(
filename='annotation/test.list',
batch_size=1,
start_pos=-1,
num_frames_per_clip=c3d_model.NUM_FRAMES_PER_CLIP,
crop_size=c3d_model.CROP_SIZE,
video_list=[])
acc, predict = sess.run(
[accuracy, predictions],
feed_dict={
images_placeholder: test_images,
labels_placeholder: test_labels,
keep_pro: 1
})
print('acc: {}'.format(acc))
print('predict: {}'.format(np.reshape(predict[1], [32])))
predict_list.append(np.reshape(predict[1], [32]))
print('labels: {}'.format(np.reshape(test_labels, [32])))
label_list.append(np.reshape(test_labels, [32]))
np.save('./test/predict', predict_list)
np.save('./test/label', label_list)
def conv_bn_relu(x,
out_channels,
ksize,
stride=1,
groups=1,
qweight=False,
qactivation=False,
padding='SAME',
scale=None,
has_bn=True,
has_relu=True,
phase_train=False,
scope=None):
node = {'input': x, 'output': None, 'W': None, 'b': None}
cfg_node = {
'name': scope,
'type': 'Conv2D',
'out': out_channels,
'in': 0,
'ksize': ksize,
'stride': stride,
'groups': groups,
'padding': padding,
'active': has_relu
}
with tf.variable_scope(scope):
in_channels = x.shape.as_list()[3]
cfg_node['in'] = in_channels
assert in_channels % groups == 0 and out_channels % groups == 0
shape = [ksize, ksize, in_channels // groups, out_channels]
kernel = _variable_with_weight_decay('W', shape)
tf.add_to_collection('params', kernel)
node['W'] = kernel
if qweight:
kernel = int_quantize(kernel,
scale[scope]['W'],
num_bits=8,
phase_train=phase_train)
if groups == 1:
f = tf.nn.conv2d(x,
kernel, [1, stride, stride, 1],
padding=padding)
else:
if out_channels == groups and in_channels == groups:
f = tf.nn.depthwise_conv2d(x,
tf.transpose(kernel, (0, 1, 3, 2)),
[1, stride, stride, 1],
padding=padding)
else:
kernel_list = tf.split(kernel, groups, axis=3)
x_list = tf.split(x, groups, axis=3)
f = tf.concat([
tf.nn.conv2d(x_list[i],
kernel_list[i], [1, stride, stride, 1],
padding=padding) for i in range(groups)
],
axis=3)
if has_bn:
f, bn_info = batch_norm_for_conv(f, phase_train)
_, moving_mean, moving_variance, beta, gamma = bn_info
s = gamma / tf.sqrt(moving_variance + cfg.bn_eps)
node['W'] = kernel * tf.reshape(s, (1, 1, 1, -1))
node['b'] = beta - s * moving_mean
else:
biases = _variable_on_cpu('b', out_channels,
tf.constant_initializer(0.0))
tf.add_to_collection('params', biases)
node['b'] = biases
f = tf.nn.bias_add(f, biases)
if has_relu:
f = tf.nn.relu6(f)
node['output'] = f
print(scope, f.shape)
tf.add_to_collection('nodes', node)
tf.add_to_collection('cfg_nodes', cfg_node)
if qactivation:
f = int_quantize(f,
scale[scope]['output'],
num_bits=8,
phase_train=phase_train)
return f
def run_training():
if not os.path.exists(model_save_dir):
os.makedirs(model_save_dir)
use_pretrained_model = True
model_filename = "./sports1m_finetuning_ucf101.model"
with tf.Graph().as_default():
global_step = tf.get_variable(
'global_step',
[],
initializer=tf.constant_initializer(0),
trainable=False
)
with tf.variable_scope('var_name') as var_scope:
weights = {
'wc1': _variable_with_weight_decay('wc1', [3, 3, 3, 3, 64], 0.005),
'wc2': _variable_with_weight_decay('wc2', [3, 3, 3, 64, 128], 0.005),
'wc3a': _variable_with_weight_decay('wc3a', [3, 3, 3, 128, 256], 0.005),
'wc3b': _variable_with_weight_decay('wc3b', [3, 3, 3, 256, 256], 0.005),
'wc4a': _variable_with_weight_decay('wc4a', [3, 3, 3, 256, 512], 0.005),
'wc4b': _variable_with_weight_decay('wc4b', [3, 3, 3, 512, 512], 0.005),
'wc5a': _variable_with_weight_decay('wc5a', [3, 3, 3, 512, 512], 0.005),
'wc5b': _variable_with_weight_decay('wc5b', [3, 3, 3, 512, 512], 0.005),
#'wd1': _variable_with_weight_decay('wd1', [8192, 4096], 0.005),
#'wd2': _variable_with_weight_decay('wd2', [4096, 4096], 0.005),
#'out': _variable_with_weight_decay('wout', [4096, c3d_model.NUM_CLASSES], 0.005)
}
biases = {
'bc1': _variable_with_weight_decay('bc1', [64], 0.000),
'bc2': _variable_with_weight_decay('bc2', [128], 0.000),
'bc3a': _variable_with_weight_decay('bc3a', [256], 0.000),
'bc3b': _variable_with_weight_decay('bc3b', [256], 0.000),
'bc4a': _variable_with_weight_decay('bc4a', [512], 0.000),
'bc4b': _variable_with_weight_decay('bc4b', [512], 0.000),
'bc5a': _variable_with_weight_decay('bc5a', [512], 0.000),
'bc5b': _variable_with_weight_decay('bc5b', [512], 0.000),
#'bd1': _variable_with_weight_decay('bd1', [4096], 0.000),
#'bd2': _variable_with_weight_decay('bd2', [4096], 0.000),
#'out': _variable_with_weight_decay('bout', [c3d_model.NUM_CLASSES], 0.000),
}
fcn_weights = {
'wconv6': _variable_with_weight_decay('conv6', [1, 4, 4, 512, 512], 0.005),
'wconv7': _variable_with_weight_decay('conv7', [1, 7, 7, 512, 512], 0.005),
'wup6': _variable_with_weight_decay('up6', [2, 1, 1, 4096, 512], 0.005),
'wup7': _variable_with_weight_decay('up7', [2, 1, 1, 4096, 4096], 0.005),
'wup8': _variable_with_weight_decay('up8', [2, 1, 1, fcn_model.NUM_CLASSES, 4096], 0.005),
}
fcn_biases = {
'bconv6': _variable_with_weight_decay('bconv6', [512], 0.000),
'bconv7': _variable_with_weight_decay('bconv7', [512], 0.000),
'bup6': _variable_with_weight_decay('bup6', [4096], 0.000),
'bup7': _variable_with_weight_decay('bup7', [4096], 0.000),
'bup8': _variable_with_weight_decay('bup8', [fcn_model.NUM_CLASSES], 0.000),
}
with tf.name_scope('inputs'):
images_placeholder, labels_placeholder, keep_pro = placeholder_inputs( FLAGS.batch_size )
varlist1 = list( set(fcn_weights.values() + fcn_biases.values()) )
varlist2 = list( set(weights.values() + biases.values()) )
feature_map = c3d_model.inference_c3d(
images_placeholder,
keep_pro,
FLAGS.batch_size,
weights,
biases
)
logit=fcn_model.inference_pool54(
feature_map,
keep_pro,
FLAGS.batch_size,
fcn_weights,
fcn_biases
)
loss = fcn_model_loss(
logit,
labels_placeholder,
FLAGS.batch_size
)
SGD_cdc = tf.train.GradientDescentOptimizer(1e-4).minimize(loss, var_list = varlist1)
SGD_c3d = tf.train.GradientDescentOptimizer(1e-5).minimize(loss, var_list = varlist2)
accuracy = tower_acc(logit, labels_placeholder, FLAGS.batch_size)
tf.summary.scalar('accuracy', accuracy)
variable_averages = tf.train.ExponentialMovingAverage(MOVING_AVERAGE_DECAY)
variables_averages_op = variable_averages.apply(tf.trainable_variables())
train_op = tf.group(SGD_cdc, SGD_c3d, variables_averages_op)
null_op = tf.no_op()
# Create a saver for writing training checkpoints.
saver = tf.train.Saver(weights.values() + biases.values())
new_saver = tf.train.Saver(weights.values() + biases.values()+ fcn_weights.values() + fcn_biases.values())
init = tf.global_variables_initializer()
# Create a session for running Ops on the Graph.
sess = tf.Session(
config=tf.ConfigProto(allow_soft_placement=True)
)
sess.run(init)
merged = tf.summary.merge_all()
if os.path.isfile(model_filename) and use_pretrained_model:
print 'loading pretrained_model....'
saver.restore(sess, model_filename)
print 'complete!'
# Create summary writter
train_writer = tf.summary.FileWriter('./visual_logs/SGD_pool54_visual_logs/train', sess.graph)
test_writer = tf.summary.FileWriter('./visual_logs/SGD_pool54_visual_logs/test', sess.graph)
video_list = []
position = -1
for step in xrange(FLAGS.max_steps+1):
start_time = time.time()
train_images, train_labels, _, _, video_list, position = input_train_data.read_clip_and_label(
filename='annotation/train.list',
batch_size=FLAGS.batch_size,
start_pos=position,
num_frames_per_clip=c3d_model.NUM_FRAMES_PER_CLIP,
crop_size=c3d_model.CROP_SIZE,
video_list=video_list
)
sess.run(train_op, feed_dict={
images_placeholder: train_images,
labels_placeholder: train_labels,
keep_pro: 0.5
})
duration = time.time() - start_time
print('Batchnum %d: %.3f sec' % (step, duration))
if (step) %2 == 0 or (step + 1) == FLAGS.max_steps:
print('Step %d/%d: %.3f sec' % (step, FLAGS.max_steps, duration))
print('Training Data Eval:')
summary,loss_train,acc = sess.run(
[merged, loss, accuracy],
feed_dict={
images_placeholder: train_images,
labels_placeholder: train_labels,
keep_pro: 1
})
print 'loss: %f' % np.mean(loss_train)
print ("accuracy: " + "{:.5f}".format(acc))
train_writer.add_summary(summary, step)
if (step) %10 == 0 or (step + 1) == FLAGS.max_steps:
print('Validation Data Eval:')
val_images, val_labels, _, _, _, _ = input_train_data.read_clip_and_label(
filename='annotation/test.list',
batch_size=FLAGS.batch_size,
start_pos=-1,
num_frames_per_clip=c3d_model.NUM_FRAMES_PER_CLIP,
crop_size=c3d_model.CROP_SIZE,
video_list=[]
)
summary,loss_val, acc = sess.run(
[merged, loss, accuracy],
feed_dict={
images_placeholder: val_images,
labels_placeholder: val_labels,
keep_pro: 1
})
print 'loss: %f' % np.mean(loss_val)
print ("accuracy: " + "{:.5f}".format(acc))
test_writer.add_summary(summary, step)
# Save the model checkpoint periodically.
if step > 1 and step % 200 == 0:
checkpoint_path = os.path.join('./models/SGD_pool54', 'model.ckpt')
new_saver.save(sess, checkpoint_path, global_step=global_step)
print("done")
def setup_graph(
self, images, phase_train
): # previous inference() labels,inference, batch_size -- in order to get batch_size at running time
#rather than using fixed batch_size in graph set up, revise it in inference:
batchsize = tf.shape(images)[0] # yike !!!
print('GGG')
print(images.get_shape())
# norm1
norm1 = tf.nn.lrn(images,
depth_radius=5,
bias=1.0,
alpha=0.0001,
beta=0.75,
name='norm1')
print(norm1.get_shape())
# conv1
conv1 = conv_layer_with_bn(
norm1, [7, 7, images.get_shape().as_list()[3], 64],
phase_train,
name="conv1") # yike: 7 too large? how about 3?
print(conv1.get_shape())
# pool1
pool1, pool1_indices = tf.nn.max_pool_with_argmax(conv1,
ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1],
padding='SAME',
name='pool1')
print('111111')
print(pool1.get_shape())
print(pool1_indices.get_shape())
# conv2
conv2 = conv_layer_with_bn(pool1, [7, 7, 64, 64],
phase_train,
name="conv2")
# pool2
pool2, pool2_indices = tf.nn.max_pool_with_argmax(conv2,
ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1],
padding='SAME',
name='pool2')
print('22222')
print(pool2.get_shape())
print(pool2_indices.get_shape())
# conv3
conv3 = conv_layer_with_bn(pool2, [7, 7, 64, 64],
phase_train,
name="conv3")
# pool3
pool3, pool3_indices = tf.nn.max_pool_with_argmax(conv3,
ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1],
padding='SAME',
name='pool3')
print('33333')
print(pool3.get_shape())
print(pool3_indices.get_shape())
# conv4
conv4 = conv_layer_with_bn(pool3, [7, 7, 64, 64],
phase_train,
name="conv4")
# pool4
pool4, pool4_indices = tf.nn.max_pool_with_argmax(conv4,
ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1],
padding='SAME',
name='pool4')
print('44444')
print(pool4.get_shape())
print(pool4_indices.get_shape())
""" End of encoder """
""" start upsample """
# upsample4
# Need to change when using different dataset out_w, out_h
# upsample4 = upsample_with_pool_indices(pool4, pool4_indices, pool4.get_shape(), out_w=45, out_h=60, scale=2, name='upsample4')
pool3_shape = pool3.get_shape()
upsample4 = deconv_layer(
pool4, [2, 2, 64, 64],
tf.stack([batchsize, pool3_shape[1], pool3_shape[2],
64]), 2, "up4") #45, 60,
#concat 4 yike
#combined4=tf.concat(axis=3,values=(upsample4,pool3))
combined4 = tf.concat(axis=3, values=(upsample4, conv4))
#print(tf.stack([batchsize, 45, 60, 64]))
# decode 4
conv_decode4 = conv_layer_with_bn(combined4, [7, 7, 128, 64],
phase_train,
False,
name="conv_decode4")
print('d4444444')
print(conv_decode4.get_shape())
# upsample 3
# upsample3 = upsample_with_pool_indices(conv_decode4, pool3_indices, conv_decode4.get_shape(), scale=2, name='upsample3')
pool2_shape = pool2.get_shape()
upsample3 = deconv_layer(
conv_decode4, [2, 2, 64, 64],
tf.stack([batchsize, pool2_shape[1], pool2_shape[2],
64]), 2, "up3") #90, 120
#concat 3 yike
# combined3=tf.concat(axis=3,values=(upsample3,pool2))
combined3 = tf.concat(axis=3, values=(upsample3, conv3))
# decode 3
conv_decode3 = conv_layer_with_bn(combined3, [7, 7, 128, 64],
phase_train,
False,
name="conv_decode3")
print('d333333')
print(conv_decode3.get_shape())
# upsample2
# upsample2 = upsample_with_pool_indices(conv_decode3, pool2_indices, conv_decode3.get_shape(), scale=2, name='upsample2')
pool1_shape = pool1.get_shape()
upsample2 = deconv_layer(
conv_decode3, [2, 2, 64, 64],
tf.stack([batchsize, pool1_shape[1], pool1_shape[2],
64]), 2, "up2") #180, 240
#concat 2 yike
#combined2=tf.concat(axis=3,values=(upsample2,pool1))
combined2 = tf.concat(axis=3, values=(upsample2, conv2))
# decode 2
conv_decode2 = conv_layer_with_bn(combined2, [7, 7, 128, 64],
phase_train,
False,
name="conv_decode2")
print('d22222')
print(conv_decode2.get_shape())
# upsample1
# upsample1 = upsample_with_pool_indices(conv_decode2, pool1_indices, conv_decode2.get_shape(), scale=2, name='upsample1')
upsample1 = deconv_layer(
conv_decode2, [2, 2, 64, 64],
tf.stack([batchsize, self.args.image_h, self.args.image_w,
64]), 2, "up1"
) # IMAGE_HEIGHT, IMAGE_WIDTH yike !!!! deconv_layer(conv_decode2, [2, 2, 64, 64], [batch_size, 360, 480, 64], 2, "up1")
#concat 1 yike
#combined2=tf.concat(axis=3,values=(upsample2,pool1))
combined1 = tf.concat(axis=3, values=(upsample1, conv1))
# decode4
conv_decode1 = conv_layer_with_bn(combined1, [7, 7, 128, 64],
phase_train,
False,
name="conv_decode1")
print('d111111')
print(conv_decode1.get_shape())
""" end of Decode """
""" Start Classify """
# output predicted class number (6)
with tf.variable_scope('conv_classifier') as scope:
kernel = _variable_with_weight_decay(
'weights',
shape=[1, 1, 64, self.num_classes],
initializer=msra_initializer(1, 64),
wd=0.0005)
conv = tf.nn.conv2d(conv_decode1,
kernel, [1, 1, 1, 1],
padding='SAME')
print('cv')
print(conv.get_shape())
biases = _variable_on_cpu('biases', [self.num_classes],
tf.constant_initializer(0.0))
print(biases.get_shape())
logit = tf.nn.bias_add(conv, biases, name=scope.name)
#conv_classifier = tf.nn.bias_add(conv, biases, name=scope.name)
#print(conv_classifier.get_shape())
#logit = conv_classifier
#print('LLL')
#print(labels)
#print(conv_classifier)
#loss = cal_loss(conv_classifier, labels)
print(logit.get_shape())
return logit # loss
def inception_v1_module(feed,
feed_dim=256,
map_size=(128, 192, 96, 64),
reduce1x1_size=64,
batch_norm=False):
"""
:param feed:
:param map_size: lists of number of feature maps output by each tower (1x1, 3x3, 5x5, 1x1) inside the Inception module
:param reduce1x1_size: number of feature maps output by each 1×1 convolution that precedes a large convolution
:return:
"""
def conv2d_s1(x, W):
return conv2d(x, W, strides=[1, 1, 1, 1], padding='SAME')
def max_pool_3x3_s1(x):
return tf.nn.max_pool(x,
ksize=[1, 3, 3, 1],
strides=[1, 1, 1, 1],
padding='SAME')
# follows input
W_conv_1x1_1 = _variable_with_weight_decay(
'W_conv_1x1_1',
shape=[1, 1, feed_dim, map_size[0]],
stddev=5e-2,
wd=None)
b_conv_1x1_1 = _variable_on_cpu('b_conv_1x1_1', [map_size[0]],
tf.constant_initializer(0.0))
# follows input
W_conv_1x1_2 = _variable_with_weight_decay(
'W_conv_1x1_2',
shape=[1, 1, feed_dim, reduce1x1_size],
stddev=5e-2,
wd=None)
b_conv_1x1_2 = _variable_on_cpu('b_conv_1x1_2', [reduce1x1_size],
tf.constant_initializer(0.0))
# follows input
W_conv_1x1_3 = _variable_with_weight_decay(
'W_conv_1x1_3',
shape=[1, 1, feed_dim, reduce1x1_size],
stddev=5e-2,
wd=None)
b_conv_1x1_3 = _variable_on_cpu('b_conv_1x1_3', [reduce1x1_size],
tf.constant_initializer(0.0))
# follows 1x1_2
# attention to the shape paras!!!!
W_conv_3x3 = _variable_with_weight_decay(
'W_conv_3x3',
shape=[3, 3, reduce1x1_size, map_size[1]],
stddev=5e-2,
wd=None)
b_conv_3x3 = _variable_on_cpu('b_conv_3x3', [map_size[1]],
tf.constant_initializer(0.0))
# follows 1x1_3
W_conv_5x5 = _variable_with_weight_decay(
'W_conv_5x5',
shape=[5, 5, reduce1x1_size, map_size[2]],
stddev=5e-2,
wd=None)
b_conv_5x5 = _variable_on_cpu('b_conv_5x5', [map_size[2]],
tf.constant_initializer(0.0))
# follows max pooling
W_conv_1x1_4 = _variable_with_weight_decay(
'W_conv_1x1_4',
shape=[1, 1, feed_dim, map_size[3]],
stddev=5e-2,
wd=None)
b_conv_1x1_4 = _variable_on_cpu('b_conv_1x1_4', [map_size[3]],
tf.constant_initializer(0.0))
# Inception Module
conv_1x1_1 = conv2d_s1(feed, W_conv_1x1_1) + b_conv_1x1_1
conv_1x1_2 = tf.nn.relu(conv2d_s1(feed, W_conv_1x1_2) + b_conv_1x1_2)
conv_1x1_3 = tf.nn.relu(conv2d_s1(feed, W_conv_1x1_3) + b_conv_1x1_3)
conv_3x3 = conv2d_s1(conv_1x1_2, W_conv_3x3) + b_conv_3x3
conv_5x5 = conv2d_s1(conv_1x1_3, W_conv_5x5) + b_conv_5x5
maxpool1 = max_pool_3x3_s1(feed)
conv_1x1_4 = conv2d_s1(maxpool1, W_conv_1x1_4) + b_conv_1x1_4
# concatenate all the feature maps and hit them with a relu
concat = tf.concat([conv_1x1_1, conv_3x3, conv_5x5, conv_1x1_4], 3)
if batch_norm:
biases = _variable_on_cpu('biases', sum(map_size),
tf.constant_initializer(0.0))
mean, variance = tf.nn.moments(concat, axes=[0])
epsilon = 1e-5
gamma = _variable_on_cpu('gammas', sum(map_size),
tf.constant_initializer(1.0))
pre_activation = tf.nn.batch_normalization(concat, mean, variance,
biases, gamma, epsilon)
else:
pre_activation = concat
after_activation = tf.nn.relu(pre_activation, name='activated_out')
_activation_summary(after_activation)
return after_activation
