def model_fn(x, target, mode, params):
"""Model function for Estimator."""
y_ = tf.cast(target, tf.float32)
x_image = tf.reshape(x, [-1, 28, 28, 1])
# first convolutional layer
h_conv1 = layers.convolution2d(x_image, 32, [5,5])
h_pool1 = layers.max_pool2d(h_conv1, [2,2])
# second convolutional layer
h_conv2 = layers.convolution2d(h_pool1, 64, [5,5])
h_pool2 = layers.max_pool2d(h_conv2, [2,2])
# densely connected layer
h_pool2_flat = tf.reshape(h_pool2, [-1, 7*7*64])
h_fc1 = layers.fully_connected(h_pool2_flat, 1024)
h_fc1_drop = layers.dropout(
h_fc1, keep_prob=params["dropout"],
is_training=(mode == ModeKeys.TRAIN))
# readout layer
y_conv = layers.fully_connected(h_fc1_drop, 10, activation_fn=None)
cross_entropy = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(y_conv, y_))
train_op = tf.contrib.layers.optimize_loss(
loss=cross_entropy,
global_step=tf.contrib.framework.get_global_step(),
learning_rate=params["learning_rate"],
optimizer="Adam")
predictions = tf.argmax(y_conv, 1)
return predictions, cross_entropy, train_op
def general_module_end_operations(self, tensor, dropout_on, strided_max_pool_on):
"""
Common end of module operations.
:param tensor: The tensor being processed.
:type tensor: tf.Tensor
:param dropout_on: Whether to include dropout or not.
:type dropout_on: bool
:param strided_max_pool_on: Whether to include a strided max pool at the end of the module.
:type strided_max_pool_on: bool
:return: The processed tensor.
:rtype: tf.Tensor
"""
if strided_max_pool_on:
tensor = max_pool2d(tensor, kernel_size=3, stride=2, padding='VALID')
if dropout_on:
tensor = dropout(tensor, self.dropout_keep_probability_tensor)
return tensor
def mercury_module(self, variable_scope, input_tensor, aisle_convolution_depth, spatial_convolution_depth,
max_pool_depth, dropout_on=False, normalization_function=default_normalization,
activation_function=default_activation, strided_max_pool_on=False):
"""
This module has 4 parts. A simple 1x1 dimensionality shift (the aisle convolution), a 1x3 convolution, a 3x1
convolution, and a 2x2 max pooling with dimensionality shift. All have stride of 1. The outputs of each part are
concatenated to form an output tensor.
:param variable_scope: What to name the module scope in the graph.
:type variable_scope: str
:param input_tensor: The input tensor to work on.
:type input_tensor: tf.Tensor
:param aisle_convolution_depth: The output depth of the 1x1 convolution.
:type aisle_convolution_depth: int
:param spatial_convolution_depth: The output depth of the 1x3 and 3x1 convolutions (each).
:type spatial_convolution_depth: int
:param max_pool_depth: The output depth of the (dimensional shifted) max pool.
:type max_pool_depth: int
:param dropout_on: A boolean to choose whether or not dropout should be applied.
:type dropout_on: bool
:param normalization_function: A normalization to be applied before activations. Defaults to batch_norm.
:type normalization_function: tf.Tensor -> tf.Tensor
:param activation_function: The activation function to be applied.
:type activation_function: tf.Tensor -> tf.Tensor
:param strided_max_pool_on: Whether to include a strided max pool at the end of the module.
:type strided_max_pool_on: bool
:return: The output activation tensor.
:rtype: tf.Tensor
"""
with tf.variable_scope(variable_scope):
part1 = convolution2d(input_tensor, aisle_convolution_depth, [1, 1], activation_fn=activation_function,
normalizer_fn=normalization_function)
part2 = convolution2d(input_tensor, spatial_convolution_depth, [3, 1], activation_fn=activation_function,
normalizer_fn=normalization_function)
part3 = convolution2d(input_tensor, spatial_convolution_depth, [1, 3], activation_fn=activation_function,
normalizer_fn=normalization_function)
max_pool_output = max_pool2d(input_tensor, kernel_size=2, stride=1, padding='SAME')
part4 = convolution2d(max_pool_output, max_pool_depth, [1, 1], activation_fn=activation_function,
normalizer_fn=normalization_function)
output_tensor = tf.concat(axis=3, values=[part1, part2, part3, part4])
output_tensor = self.general_module_end_operations(output_tensor, dropout_on, strided_max_pool_on)
return output_tensor
def model_fn(self, features, labels, mode):
"""Dense neural network with 0 hidden layer.
Flatten then dense. Can be applied to any task. Here we apply it to speech
and tabular data.
"""
col_count, row_count = self.metadata_.get_matrix_size(0)
sequence_size = self.metadata_.get_sequence_size()
output_dim = self.metadata_.get_output_size()
# Construct a neural network with 0 hidden layer
input_layer = tf.reshape(features["x"],
[-1, sequence_size*row_count*col_count])
# Replace missing values by 0
input_layer = tf.where(tf.is_nan(input_layer),
tf.zeros_like(input_layer), input_layer)
input_layer = tf.divide(input_layer,np.iinfo(np.int16).max)
specgram = signal.stft(input_layer, 400, 160)
phase = tf.angle(specgram) / np.pi
amp = tf.log1p(tf.abs(specgram))
x = tf.stack([amp, phase], axis=3) # shape is [bs, time, freq_bins, 2]
x = tf.to_float(x)
x = layers.batch_norm(x, is_training=(mode == tf.estimator.ModeKeys.TRAIN))
for i in range(4):
x = layers.conv2d(
x, 16 * (2 ** i), 3, 1,
activation_fn=tf.nn.elu,
normalizer_fn=layers.batch_norm if True else None,
normalizer_params={'is_training': (mode == tf.estimator.ModeKeys.TRAIN)}
)
x = layers.max_pool2d(x, 2, 2)
mpool = tf.reduce_max(x, axis=[1, 2], keep_dims=True)
apool = tf.reduce_mean(x, axis=[1, 2], keep_dims=True)
x = 0.5 * (mpool + apool)
x = tf.layers.flatten(x)
x = tf.layers.dense(inputs=x, units=128, activation=tf.nn.elu)
x = tf.nn.dropout(x, keep_prob=0.5 if (mode == tf.estimator.ModeKeys.TRAIN) else 1.0)
logits = tf.layers.dense(inputs=x, units=output_dim)
# For multi-label classification, the correct loss is actually sigmoid with
# sigmoid_cross_entropy_with_logits, not softmax with softmax_cross_entropy.
#softmax_tensor = tf.nn.softmax(logits, name="softmax_tensor")
sigmoid_tensor = tf.nn.sigmoid(logits, name="sigmoid_tensor")
predictions = {
# Generate predictions (for PREDICT and EVAL mode)
"classes": tf.argmax(input=logits, axis=1),
# "classes": binary_predictions,
# Add `softmax_tensor` to the graph. It is used for PREDICT and by the
# `logging_hook`.
#"probabilities": softmax_tensor
"probabilities": sigmoid_tensor
}
if mode == tf.estimator.ModeKeys.PREDICT:
return tf.estimator.EstimatorSpec(mode=mode, predictions=predictions)
# Calculate Loss (for both TRAIN and EVAL modes)
#loss = tf.losses.softmax_cross_entropy(onehot_labels=labels, logits=logits)
loss = sigmoid_cross_entropy_with_logits(labels=labels, logits=logits)
# Configure the Training Op (for TRAIN mode)
if mode == tf.estimator.ModeKeys.TRAIN:
optimizer = tf.train.AdamOptimizer()
train_op = optimizer.minimize(
loss=loss,
global_step=tf.train.get_global_step())
return tf.estimator.EstimatorSpec(mode=mode, loss=loss, train_op=train_op)
# Add evaluation metrics (for EVAL mode)
assert mode == tf.estimator.ModeKeys.EVAL
eval_metric_ops = {
"accuracy": tf.metrics.accuracy(
labels=labels, predictions=predictions["classes"])}
return tf.estimator.EstimatorSpec(
mode=mode, loss=loss, eval_metric_ops=eval_metric_ops)
cross_entropy = tf.reduce_mean(-tf.reduce_sum(y_ * tf.log(y), reduction_indices=[1]))'''
x_image = tf.reshape(x, [-1, 28, 28, 1])
stack1_conv1 = layers.convolution2d(
x_image,
64, [3, 3],
weights_regularizer=layers.l2_regularizer(0.1),
biases_regularizer=layers.l2_regularizer(0.1),
scope='stack1_Conv1')
stack1_conv2 = layers.convolution2d(
stack1_conv1,
64, [3, 3],
weights_regularizer=layers.l2_regularizer(0.1),
biases_regularizer=layers.l2_regularizer(0.1),
scope='stack1_Conv2')
stack1_pool = layers.max_pool2d(stack1_conv2, [2, 2],
padding='SAME',
scope='stack1_Pool')
stack3_pool_flat = layers.flatten(stack1_pool,
scope='stack3_pool_flat')
fcl1 = layers.fully_connected(
stack3_pool_flat,
512,
weights_regularizer=layers.l2_regularizer(0.1),
biases_regularizer=layers.l2_regularizer(0.1),
scope='FCL1')
fcl1_d = layers.dropout(fcl1, keep_prob=0.5, scope='dropout1')
fcl2 = layers.fully_connected(
fcl1_d,
128,
weights_regularizer=layers.l2_regularizer(0.1),
biases_regularizer=layers.l2_regularizer(0.1),
def max_pool2d(input, kernel_size=3, stride=2, padding='VALID', name=None, data_format='NHWC'):
return contrib_layers.max_pool2d(input, kernel_size=kernel_size, stride=stride, padding=padding, scope=name, data_format=data_format)
def build(self, fully_connected_layers, inputs, labels, weight_decay,
vgg_init_dir, is_training):
bn_params = {
'decay': 0.999,
'center': True,
'scale': True,
'epsilon': 0.001,
'updates_collections': None,
'is_training': is_training,
}
if is_training:
vgg_layers, vgg_layer_names = read_vgg_init(vgg_init_dir)
inputs_shape = inputs.get_shape()
horizontal_slice_size = int(round(int(inputs_shape[2]) / 3))
vertical_slice_size = int(round(int(inputs_shape[1]) / 3))
inputs = tf.slice(inputs,
begin=[0, vertical_slice_size, 0, 0],
size=[-1, -1, horizontal_slice_size * 2, -1])
with tf.contrib.framework.arg_scope(
[layers.convolution2d],
kernel_size=3,
stride=1,
padding='SAME',
rate=1,
activation_fn=tf.nn.relu,
normalizer_fn=None,
weights_initializer=None,
weights_regularizer=layers.l2_regularizer(weight_decay)):
net = layers.convolution2d(inputs, 64, scope='conv1_1')
net = layers.convolution2d(net, 64, scope='conv1_2')
net = layers.max_pool2d(net, 2, 2, scope='pool1')
net = layers.convolution2d(net, 128, scope='conv2_1')
net = layers.convolution2d(net, 128, scope='conv2_2')
net = layers.max_pool2d(net, 2, 2, scope='pool2')
net = layers.convolution2d(net, 256, scope='conv3_1')
net = layers.convolution2d(net, 256, scope='conv3_2')
net = layers.convolution2d(net, 256, scope='conv3_3')
net = layers.max_pool2d(net, 2, 2, scope='pool3')
net = layers.convolution2d(net, 512, scope='conv4_1')
net = layers.convolution2d(net, 512, scope='conv4_2')
net = layers.convolution2d(net, 512, scope='conv4_3')
net = layers.max_pool2d(net, 2, 2, scope='pool4')
net = layers.convolution2d(net, 512, scope='conv5_1')
net = layers.convolution2d(net, 512, scope='conv5_2')
net = layers.convolution2d(net, 512, scope='conv5_3')
net = layers.max_pool2d(net, 2, 2, scope='pool5')
if is_training:
init_op, init_feed, pretrained_vars = create_init_op(vgg_layers)
self.pretrained_vars = pretrained_vars
"""
net = layers.batch_norm(net, decay=bn_params['decay'], center=bn_params['center'],
scale=bn_params['scale'], epsilon=bn_params['epsilon'],
updates_collections=bn_params['updates_collections'], is_training=bn_params['is_training'],
scope='batch_norm'
)
"""
net_shape = net.get_shape()
batch_size = tf.shape(inputs)[0]
net = tf.reshape(net, [
batch_size,
int(net_shape[1]) * int(net_shape[2]) * int(net_shape[3])
])
with tf.contrib.framework.arg_scope(
[layers.fully_connected],
activation_fn=tf.nn.relu,
normalizer_fn=layers.batch_norm,
normalizer_params=bn_params,
weights_initializer=layers.variance_scaling_initializer(),
weights_regularizer=layers.l2_regularizer(weight_decay)):
layer_num = 1
for fully_connected_num in fully_connected_layers:
net = layers.fully_connected(net,
fully_connected_num,
scope='fc{}'.format(layer_num))
layer_num += 1
logits = layers.fully_connected(
net,
2,
activation_fn=None,
weights_initializer=layers.xavier_initializer(),
weights_regularizer=layers.l2_regularizer(weight_decay),
biases_initializer=tf.zeros_initializer(),
scope='logits')
total_loss = loss(logits, labels, is_training)
if is_training:
return logits, total_loss, init_op, init_feed
else:
return logits, total_loss
def build(self, lstm_state_sizes, inputs, labels, weight_decay,
vgg_init_dir, is_training):
if is_training:
vgg_layers, vgg_layer_names = read_vgg_init(vgg_init_dir)
batch_size = tf.shape(inputs)[0]
inputs_shape = inputs.get_shape()
horizontal_slice_size = int(round(int(inputs_shape[3]) / 3))
vertical_slice_size = int(round(int(inputs_shape[2]) / 3))
inputs = tf.slice(inputs,
begin=[0, 0, vertical_slice_size, 0, 0],
size=[-1, -1, -1, horizontal_slice_size * 2, -1])
concated = None
sequence_length = int(inputs_shape[1])
reuse = None
for sequence_image in range(sequence_length):
with tf.contrib.framework.arg_scope(
[layers.convolution2d],
kernel_size=3,
stride=1,
padding='SAME',
rate=1,
activation_fn=tf.nn.relu,
normalizer_fn=None,
weights_initializer=None,
weights_regularizer=layers.l2_regularizer(weight_decay)):
net = layers.convolution2d(inputs[:, sequence_image],
64,
scope='conv1_1',
reuse=reuse)
net = layers.convolution2d(net,
64,
scope='conv1_2',
reuse=reuse)
net = layers.max_pool2d(net, 2, 2, scope='pool1')
net = layers.convolution2d(net,
128,
scope='conv2_1',
reuse=reuse)
net = layers.convolution2d(net,
128,
scope='conv2_2',
reuse=reuse)
net = layers.max_pool2d(net, 2, 2, scope='pool2')
net = layers.convolution2d(net,
256,
scope='conv3_1',
reuse=reuse)
net = layers.convolution2d(net,
256,
scope='conv3_2',
reuse=reuse)
net = layers.convolution2d(net,
256,
scope='conv3_3',
reuse=reuse)
net = layers.max_pool2d(net, 2, 2, scope='pool3')
net = layers.convolution2d(net,
512,
scope='conv4_1',
reuse=reuse)
net = layers.convolution2d(net,
512,
scope='conv4_2',
reuse=reuse)
net = layers.convolution2d(net,
512,
scope='conv4_3',
reuse=reuse)
net = layers.max_pool2d(net, 2, 2, scope='pool4')
net = layers.convolution2d(net,
512,
scope='conv5_1',
reuse=reuse)
net = layers.convolution2d(net,
512,
scope='conv5_2',
reuse=reuse)
net = layers.convolution2d(net,
512,
scope='conv5_3',
reuse=reuse)
net = layers.max_pool2d(net, 2, 2, scope='pool5')
net_shape = net.get_shape()
net = tf.reshape(net, [
batch_size,
int(net_shape[1]) * int(net_shape[2]) * int(net_shape[3])
])
if concated is None:
concated = tf.expand_dims(net, axis=0)
else:
concated = tf.concat(
[concated, tf.expand_dims(net, axis=0)], axis=0)
reuse = True
output_weights = tf.get_variable(
'lstm_output_weights',
shape=[lstm_state_sizes[-1], 2],
initializer=layers.xavier_initializer(),
regularizer=layers.l2_regularizer(weight_decay))
output_bias = tf.get_variable('lstm_output_bias',
shape=[2],
initializer=tf.zeros_initializer())
lstms = []
for index, lstm_state_size in enumerate(lstm_state_sizes):
with tf.variable_scope('lstm{}'.format(index)):
lstm_reuse = tf.get_variable_scope().reuse
lstms.append(
tf.contrib.rnn.BasicLSTMCell(lstm_state_size,
reuse=lstm_reuse))
multi_layered_lstms = tf.contrib.rnn.MultiRNNCell(lstms)
if is_training:
init_op, init_feed, pretrained_vars = create_init_op(vgg_layers)
self.pretrained_vars = pretrained_vars
net = tf.unstack(concated, num=sequence_length, axis=0)
outputs, states = tf.contrib.rnn.static_rnn(multi_layered_lstms,
net,
dtype=tf.float32)
logits = tf.matmul(outputs[-1], output_weights) + output_bias
total_loss = loss(logits, labels, is_training)
if is_training:
return logits, total_loss, init_op, init_feed
else:
return logits, total_loss
def build(self):
print('[*]Building...')
with tf.variable_scope('cnn'):
with tf.name_scope('normalize'):
image = tf.div(self.x, 255)
with tf.variable_scope('conv1') as scope:
conv1 = layers.conv2d(image, 64, 3, scope=scope)
self.layers.append(conv1)
with tf.variable_scope('pool1'):
pool1 = layers.max_pool2d(conv1, kernel_size=2, padding='SAME')
self.layers.append(pool1)
with tf.variable_scope('conv2') as scope:
conv2 = layers.conv2d(pool1,
num_outputs=64,
kernel_size=3,
scope=scope)
self.layers.append(conv2)
with tf.variable_scope('pool2'):
pool2 = layers.max_pool2d(conv2, kernel_size=2, padding='SAME')
self.layers.append(pool2)
with tf.variable_scope('conv3') as scope:
conv3 = layers.conv2d(pool2,
num_outputs=128,
kernel_size=3,
scope=scope)
self.layers.append(conv3)
with tf.variable_scope('pool3'):
pool3 = layers.max_pool2d(conv3, kernel_size=2, padding='SAME')
self.layers.append(pool3)
with tf.variable_scope('conv4') as scope:
conv4 = layers.conv2d(pool3, 128, 3, scope=scope)
self.layers.append(conv4)
with tf.variable_scope('pool4'):
pool4 = layers.max_pool2d(conv4, 2, padding='SAME')
self.layers.append(pool4)
with tf.variable_scope('fc1') as scope:
flattened = layers.flatten(pool4)
fc1 = layers.fully_connected(flattened, 256, scope=scope)
self.layers.append(fc1)
with tf.variable_scope('dropout'):
dropout = layers.dropout(fc1, is_training=self.is_training)
self.layers.append(dropout)
with tf.variable_scope('fc2') as scope:
# tanh layer to scale y
y = layers.fully_connected(dropout,
2,
scope=scope,
activation_fn=tf.nn.tanh)
self.layers.append(y)
print('[*]Building complete')
return y
def inference_person(image):
with tf.variable_scope('PersonNet'):
conv1_1 = layers.conv2d(image,
64,
3,
1,
activation_fn=None,
scope='conv1_1')
conv1_1 = tf.nn.relu(conv1_1)
conv1_2 = layers.conv2d(conv1_1,
64,
3,
1,
activation_fn=None,
scope='conv1_2')
conv1_2 = tf.nn.relu(conv1_2)
pool1_stage1 = layers.max_pool2d(conv1_2, 2, 2)
conv2_1 = layers.conv2d(pool1_stage1,
128,
3,
1,
activation_fn=None,
scope='conv2_1')
conv2_1 = tf.nn.relu(conv2_1)
conv2_2 = layers.conv2d(conv2_1,
128,
3,
1,
activation_fn=None,
scope='conv2_2')
conv2_2 = tf.nn.relu(conv2_2)
pool2_stage1 = layers.max_pool2d(conv2_2, 2, 2)
conv3_1 = layers.conv2d(pool2_stage1,
256,
3,
1,
activation_fn=None,
scope='conv3_1')
conv3_1 = tf.nn.relu(conv3_1)
conv3_2 = layers.conv2d(conv3_1,
256,
3,
1,
activation_fn=None,
scope='conv3_2')
conv3_2 = tf.nn.relu(conv3_2)
conv3_3 = layers.conv2d(conv3_2,
256,
3,
1,
activation_fn=None,
scope='conv3_3')
conv3_3 = tf.nn.relu(conv3_3)
conv3_4 = layers.conv2d(conv3_3,
256,
3,
1,
activation_fn=None,
scope='conv3_4')
conv3_4 = tf.nn.relu(conv3_4)
pool3_stage1 = layers.max_pool2d(conv3_4, 2, 2)
conv4_1 = layers.conv2d(pool3_stage1,
512,
3,
1,
activation_fn=None,
scope='conv4_1')
conv4_1 = tf.nn.relu(conv4_1)
conv4_2 = layers.conv2d(conv4_1,
512,
3,
1,
activation_fn=None,
scope='conv4_2')
conv4_2 = tf.nn.relu(conv4_2)
conv4_3 = layers.conv2d(conv4_2,
512,
3,
1,
activation_fn=None,
scope='conv4_3')
conv4_3 = tf.nn.relu(conv4_3)
conv4_4 = layers.conv2d(conv4_3,
512,
3,
1,
activation_fn=None,
scope='conv4_4')
conv4_4 = tf.nn.relu(conv4_4)
conv5_1 = layers.conv2d(conv4_4,
512,
3,
1,
activation_fn=None,
scope='conv5_1')
conv5_1 = tf.nn.relu(conv5_1)
conv5_2_CPM = layers.conv2d(conv5_1,
128,
3,
1,
activation_fn=None,
scope='conv5_2_CPM')
conv5_2_CPM = tf.nn.relu(conv5_2_CPM)
conv6_1_CPM = layers.conv2d(conv5_2_CPM,
512,
1,
1,
activation_fn=None,
scope='conv6_1_CPM')
conv6_1_CPM = tf.nn.relu(conv6_1_CPM)
conv6_2_CPM = layers.conv2d(conv6_1_CPM,
1,
1,
1,
activation_fn=None,
scope='conv6_2_CPM')
concat_stage2 = tf.concat(axis=3, values=[conv6_2_CPM, conv5_2_CPM])
Mconv1_stage2 = layers.conv2d(concat_stage2,
128,
7,
1,
activation_fn=None,
scope='Mconv1_stage2')
Mconv1_stage2 = tf.nn.relu(Mconv1_stage2)
Mconv2_stage2 = layers.conv2d(Mconv1_stage2,
128,
7,
1,
activation_fn=None,
scope='Mconv2_stage2')
Mconv2_stage2 = tf.nn.relu(Mconv2_stage2)
Mconv3_stage2 = layers.conv2d(Mconv2_stage2,
128,
7,
1,
activation_fn=None,
scope='Mconv3_stage2')
Mconv3_stage2 = tf.nn.relu(Mconv3_stage2)
Mconv4_stage2 = layers.conv2d(Mconv3_stage2,
128,
7,
1,
activation_fn=None,
scope='Mconv4_stage2')
Mconv4_stage2 = tf.nn.relu(Mconv4_stage2)
Mconv5_stage2 = layers.conv2d(Mconv4_stage2,
128,
7,
1,
activation_fn=None,
scope='Mconv5_stage2')
Mconv5_stage2 = tf.nn.relu(Mconv5_stage2)
Mconv6_stage2 = layers.conv2d(Mconv5_stage2,
128,
1,
1,
activation_fn=None,
scope='Mconv6_stage2')
Mconv6_stage2 = tf.nn.relu(Mconv6_stage2)
Mconv7_stage2 = layers.conv2d(Mconv6_stage2,
1,
1,
1,
activation_fn=None,
scope='Mconv7_stage2')
concat_stage3 = tf.concat(axis=3, values=[Mconv7_stage2, conv5_2_CPM])
Mconv1_stage3 = layers.conv2d(concat_stage3,
128,
7,
1,
activation_fn=None,
scope='Mconv1_stage3')
Mconv1_stage3 = tf.nn.relu(Mconv1_stage3)
Mconv2_stage3 = layers.conv2d(Mconv1_stage3,
128,
7,
1,
activation_fn=None,
scope='Mconv2_stage3')
Mconv2_stage3 = tf.nn.relu(Mconv2_stage3)
Mconv3_stage3 = layers.conv2d(Mconv2_stage3,
128,
7,
1,
activation_fn=None,
scope='Mconv3_stage3')
Mconv3_stage3 = tf.nn.relu(Mconv3_stage3)
Mconv4_stage3 = layers.conv2d(Mconv3_stage3,
128,
7,
1,
activation_fn=None,
scope='Mconv4_stage3')
Mconv4_stage3 = tf.nn.relu(Mconv4_stage3)
Mconv5_stage3 = layers.conv2d(Mconv4_stage3,
128,
7,
1,
activation_fn=None,
scope='Mconv5_stage3')
Mconv5_stage3 = tf.nn.relu(Mconv5_stage3)
Mconv6_stage3 = layers.conv2d(Mconv5_stage3,
128,
1,
1,
activation_fn=None,
scope='Mconv6_stage3')
Mconv6_stage3 = tf.nn.relu(Mconv6_stage3)
Mconv7_stage3 = layers.conv2d(Mconv6_stage3,
1,
1,
1,
activation_fn=None,
scope='Mconv7_stage3')
concat_stage4 = tf.concat(axis=3, values=[Mconv7_stage3, conv5_2_CPM])
Mconv1_stage4 = layers.conv2d(concat_stage4,
128,
7,
1,
activation_fn=None,
scope='Mconv1_stage4')
Mconv1_stage4 = tf.nn.relu(Mconv1_stage4)
Mconv2_stage4 = layers.conv2d(Mconv1_stage4,
128,
7,
1,
activation_fn=None,
scope='Mconv2_stage4')
Mconv2_stage4 = tf.nn.relu(Mconv2_stage4)
Mconv3_stage4 = layers.conv2d(Mconv2_stage4,
128,
7,
1,
activation_fn=None,
scope='Mconv3_stage4')
Mconv3_stage4 = tf.nn.relu(Mconv3_stage4)
Mconv4_stage4 = layers.conv2d(Mconv3_stage4,
128,
7,
1,
activation_fn=None,
scope='Mconv4_stage4')
Mconv4_stage4 = tf.nn.relu(Mconv4_stage4)
Mconv5_stage4 = layers.conv2d(Mconv4_stage4,
128,
7,
1,
activation_fn=None,
scope='Mconv5_stage4')
Mconv5_stage4 = tf.nn.relu(Mconv5_stage4)
Mconv6_stage4 = layers.conv2d(Mconv5_stage4,
128,
1,
1,
activation_fn=None,
scope='Mconv6_stage4')
Mconv6_stage4 = tf.nn.relu(Mconv6_stage4)
Mconv7_stage4 = layers.conv2d(Mconv6_stage4,
1,
1,
1,
activation_fn=None,
scope='Mconv7_stage4')
return Mconv7_stage4
def inference_pose_v2(image, center_map):
# corresponds to pose_deploy_resize.prototxt
with tf.variable_scope('PoseNet'):
pool_center_lower = layers.avg_pool2d(center_map, 9, 8, padding='SAME')
conv1_1 = layers.conv2d(image,
64,
3,
1,
activation_fn=None,
scope='conv1_1')
conv1_1 = tf.nn.relu(conv1_1)
conv1_2 = layers.conv2d(conv1_1,
64,
3,
1,
activation_fn=None,
scope='conv1_2')
conv1_2 = tf.nn.relu(conv1_2)
pool1_stage1 = layers.max_pool2d(conv1_2, 2, 2)
conv2_1 = layers.conv2d(pool1_stage1,
128,
3,
1,
activation_fn=None,
scope='conv2_1')
conv2_1 = tf.nn.relu(conv2_1)
conv2_2 = layers.conv2d(conv2_1,
128,
3,
1,
activation_fn=None,
scope='conv2_2')
conv2_2 = tf.nn.relu(conv2_2)
pool2_stage1 = layers.max_pool2d(conv2_2, 2, 2)
conv3_1 = layers.conv2d(pool2_stage1,
256,
3,
1,
activation_fn=None,
scope='conv3_1')
conv3_1 = tf.nn.relu(conv3_1)
conv3_2 = layers.conv2d(conv3_1,
256,
3,
1,
activation_fn=None,
scope='conv3_2')
conv3_2 = tf.nn.relu(conv3_2)
conv3_3 = layers.conv2d(conv3_2,
256,
3,
1,
activation_fn=None,
scope='conv3_3')
conv3_3 = tf.nn.relu(conv3_3)
conv3_4 = layers.conv2d(conv3_3,
256,
3,
1,
activation_fn=None,
scope='conv3_4')
conv3_4 = tf.nn.relu(conv3_4)
pool3_stage1 = layers.max_pool2d(conv3_4, 2, 2)
conv4_1 = layers.conv2d(pool3_stage1,
512,
3,
1,
activation_fn=None,
scope='conv4_1')
conv4_1 = tf.nn.relu(conv4_1)
conv4_2 = layers.conv2d(conv4_1,
512,
3,
1,
activation_fn=None,
scope='conv4_2')
conv4_2 = tf.nn.relu(conv4_2)
conv4_3_CPM = layers.conv2d(conv4_2,
256,
3,
1,
activation_fn=None,
scope='conv4_3_CPM')
conv4_3_CPM = tf.nn.relu(conv4_3_CPM)
conv4_4_CPM = layers.conv2d(conv4_3_CPM,
256,
3,
1,
activation_fn=None,
scope='conv4_4_CPM')
conv4_4_CPM = tf.nn.relu(conv4_4_CPM)
conv4_5_CPM = layers.conv2d(conv4_4_CPM,
256,
3,
1,
activation_fn=None,
scope='conv4_5_CPM')
conv4_5_CPM = tf.nn.relu(conv4_5_CPM)
conv4_6_CPM = layers.conv2d(conv4_5_CPM,
256,
3,
1,
activation_fn=None,
scope='conv4_6_CPM')
conv4_6_CPM = tf.nn.relu(conv4_6_CPM)
conv4_7_CPM = layers.conv2d(conv4_6_CPM,
128,
3,
1,
activation_fn=None,
scope='conv4_7_CPM')
conv4_7_CPM = tf.nn.relu(conv4_7_CPM)
conv5_1_CPM = layers.conv2d(conv4_7_CPM,
512,
1,
1,
activation_fn=None,
scope='conv5_1_CPM')
conv5_1_CPM = tf.nn.relu(conv5_1_CPM)
conv5_2_CPM = layers.conv2d(conv5_1_CPM,
15,
1,
1,
activation_fn=None,
scope='conv5_2_CPM')
concat_stage2 = tf.concat(
axis=3, values=[conv5_2_CPM, conv4_7_CPM, pool_center_lower])
Mconv1_stage2 = layers.conv2d(concat_stage2,
128,
7,
1,
activation_fn=None,
scope='Mconv1_stage2')
Mconv1_stage2 = tf.nn.relu(Mconv1_stage2)
Mconv2_stage2 = layers.conv2d(Mconv1_stage2,
128,
7,
1,
activation_fn=None,
scope='Mconv2_stage2')
Mconv2_stage2 = tf.nn.relu(Mconv2_stage2)
Mconv3_stage2 = layers.conv2d(Mconv2_stage2,
128,
7,
1,
activation_fn=None,
scope='Mconv3_stage2')
Mconv3_stage2 = tf.nn.relu(Mconv3_stage2)
Mconv4_stage2 = layers.conv2d(Mconv3_stage2,
128,
7,
1,
activation_fn=None,
scope='Mconv4_stage2')
Mconv4_stage2 = tf.nn.relu(Mconv4_stage2)
Mconv5_stage2 = layers.conv2d(Mconv4_stage2,
128,
7,
1,
activation_fn=None,
scope='Mconv5_stage2')
Mconv5_stage2 = tf.nn.relu(Mconv5_stage2)
Mconv6_stage2 = layers.conv2d(Mconv5_stage2,
128,
1,
1,
activation_fn=None,
scope='Mconv6_stage2')
Mconv6_stage2 = tf.nn.relu(Mconv6_stage2)
Mconv7_stage2 = layers.conv2d(Mconv6_stage2,
15,
1,
1,
activation_fn=None,
scope='Mconv7_stage2')
concat_stage3 = tf.concat(
axis=3, values=[Mconv7_stage2, conv4_7_CPM, pool_center_lower])
Mconv1_stage3 = layers.conv2d(concat_stage3,
128,
7,
1,
activation_fn=None,
scope='Mconv1_stage3')
Mconv1_stage3 = tf.nn.relu(Mconv1_stage3)
Mconv2_stage3 = layers.conv2d(Mconv1_stage3,
128,
7,
1,
activation_fn=None,
scope='Mconv2_stage3')
Mconv2_stage3 = tf.nn.relu(Mconv2_stage3)
Mconv3_stage3 = layers.conv2d(Mconv2_stage3,
128,
7,
1,
activation_fn=None,
scope='Mconv3_stage3')
Mconv3_stage3 = tf.nn.relu(Mconv3_stage3)
Mconv4_stage3 = layers.conv2d(Mconv3_stage3,
128,
7,
1,
activation_fn=None,
scope='Mconv4_stage3')
Mconv4_stage3 = tf.nn.relu(Mconv4_stage3)
Mconv5_stage3 = layers.conv2d(Mconv4_stage3,
128,
7,
1,
activation_fn=None,
scope='Mconv5_stage3')
Mconv5_stage3 = tf.nn.relu(Mconv5_stage3)
Mconv6_stage3 = layers.conv2d(Mconv5_stage3,
128,
1,
1,
activation_fn=None,
scope='Mconv6_stage3')
Mconv6_stage3 = tf.nn.relu(Mconv6_stage3)
Mconv7_stage3 = layers.conv2d(Mconv6_stage3,
15,
1,
1,
activation_fn=None,
scope='Mconv7_stage3')
concat_stage4 = tf.concat(
axis=3, values=[Mconv7_stage3, conv4_7_CPM, pool_center_lower])
Mconv1_stage4 = layers.conv2d(concat_stage4,
128,
7,
1,
activation_fn=None,
scope='Mconv1_stage4')
Mconv1_stage4 = tf.nn.relu(Mconv1_stage4)
Mconv2_stage4 = layers.conv2d(Mconv1_stage4,
128,
7,
1,
activation_fn=None,
scope='Mconv2_stage4')
Mconv2_stage4 = tf.nn.relu(Mconv2_stage4)
Mconv3_stage4 = layers.conv2d(Mconv2_stage4,
128,
7,
1,
activation_fn=None,
scope='Mconv3_stage4')
Mconv3_stage4 = tf.nn.relu(Mconv3_stage4)
Mconv4_stage4 = layers.conv2d(Mconv3_stage4,
128,
7,
1,
activation_fn=None,
scope='Mconv4_stage4')
Mconv4_stage4 = tf.nn.relu(Mconv4_stage4)
Mconv5_stage4 = layers.conv2d(Mconv4_stage4,
128,
7,
1,
activation_fn=None,
scope='Mconv5_stage4')
Mconv5_stage4 = tf.nn.relu(Mconv5_stage4)
Mconv6_stage4 = layers.conv2d(Mconv5_stage4,
128,
1,
1,
activation_fn=None,
scope='Mconv6_stage4')
Mconv6_stage4 = tf.nn.relu(Mconv6_stage4)
Mconv7_stage4 = layers.conv2d(Mconv6_stage4,
15,
1,
1,
activation_fn=None,
scope='Mconv7_stage4')
concat_stage5 = tf.concat(
axis=3, values=[Mconv7_stage4, conv4_7_CPM, pool_center_lower])
Mconv1_stage5 = layers.conv2d(concat_stage5,
128,
7,
1,
activation_fn=None,
scope='Mconv1_stage5')
Mconv1_stage5 = tf.nn.relu(Mconv1_stage5)
Mconv2_stage5 = layers.conv2d(Mconv1_stage5,
128,
7,
1,
activation_fn=None,
scope='Mconv2_stage5')
Mconv2_stage5 = tf.nn.relu(Mconv2_stage5)
Mconv3_stage5 = layers.conv2d(Mconv2_stage5,
128,
7,
1,
activation_fn=None,
scope='Mconv3_stage5')
Mconv3_stage5 = tf.nn.relu(Mconv3_stage5)
Mconv4_stage5 = layers.conv2d(Mconv3_stage5,
128,
7,
1,
activation_fn=None,
scope='Mconv4_stage5')
Mconv4_stage5 = tf.nn.relu(Mconv4_stage5)
Mconv5_stage5 = layers.conv2d(Mconv4_stage5,
128,
7,
1,
activation_fn=None,
scope='Mconv5_stage5')
Mconv5_stage5 = tf.nn.relu(Mconv5_stage5)
Mconv6_stage5 = layers.conv2d(Mconv5_stage5,
128,
1,
1,
activation_fn=None,
scope='Mconv6_stage5')
Mconv6_stage5 = tf.nn.relu(Mconv6_stage5)
Mconv7_stage5 = layers.conv2d(Mconv6_stage5,
15,
1,
1,
activation_fn=None,
scope='Mconv7_stage5')
concat_stage6 = tf.concat(
axis=3, values=[Mconv7_stage5, conv4_7_CPM, pool_center_lower])
Mconv1_stage6 = layers.conv2d(concat_stage6,
128,
7,
1,
activation_fn=None,
scope='Mconv1_stage6')
Mconv1_stage6 = tf.nn.relu(Mconv1_stage6)
Mconv2_stage6 = layers.conv2d(Mconv1_stage6,
128,
7,
1,
activation_fn=None,
scope='Mconv2_stage6')
Mconv2_stage6 = tf.nn.relu(Mconv2_stage6)
Mconv3_stage6 = layers.conv2d(Mconv2_stage6,
128,
7,
1,
activation_fn=None,
scope='Mconv3_stage6')
Mconv3_stage6 = tf.nn.relu(Mconv3_stage6)
Mconv4_stage6 = layers.conv2d(Mconv3_stage6,
128,
7,
1,
activation_fn=None,
scope='Mconv4_stage6')
Mconv4_stage6 = tf.nn.relu(Mconv4_stage6)
Mconv5_stage6 = layers.conv2d(Mconv4_stage6,
128,
7,
1,
activation_fn=None,
scope='Mconv5_stage6')
Mconv5_stage6 = tf.nn.relu(Mconv5_stage6)
Mconv6_stage6 = layers.conv2d(Mconv5_stage6,
128,
1,
1,
activation_fn=None,
scope='Mconv6_stage6')
Mconv6_stage6 = tf.nn.relu(Mconv6_stage6)
Mconv7_stage6 = layers.conv2d(Mconv6_stage6,
15,
1,
1,
activation_fn=None,
scope='Mconv7_stage6')
return Mconv7_stage6
def inference_pose(image, center_map):
# corresponds to pose_deploy_centerMap.prototxt
with tf.variable_scope('PoseNet'):
pool_center_lower = layers.avg_pool2d(center_map,
9,
8,
padding='VALID')
conv1_stage1 = layers.conv2d(image,
128,
9,
1,
activation_fn=None,
scope='conv1_stage1')
conv1_stage1 = tf.nn.relu(conv1_stage1)
pool1_stage1 = layers.max_pool2d(conv1_stage1, 3, 2)
conv2_stage1 = layers.conv2d(pool1_stage1,
128,
9,
1,
activation_fn=None,
scope='conv2_stage1')
conv2_stage1 = tf.nn.relu(conv2_stage1)
pool2_stage1 = layers.max_pool2d(conv2_stage1, 3, 2)
conv3_stage1 = layers.conv2d(pool2_stage1,
128,
9,
1,
activation_fn=None,
scope='conv3_stage1')
conv3_stage1 = tf.nn.relu(conv3_stage1)
pool3_stage1 = layers.max_pool2d(conv3_stage1, 3, 2)
conv4_stage1 = layers.conv2d(pool3_stage1,
32,
5,
1,
activation_fn=None,
scope='conv4_stage1')
conv4_stage1 = tf.nn.relu(conv4_stage1)
conv5_stage1 = layers.conv2d(conv4_stage1,
512,
9,
1,
activation_fn=None,
scope='conv5_stage1')
conv5_stage1 = tf.nn.relu(conv5_stage1)
conv6_stage1 = layers.conv2d(conv5_stage1,
512,
1,
1,
activation_fn=None,
scope='conv6_stage1')
conv6_stage1 = tf.nn.relu(conv6_stage1)
conv7_stage1 = layers.conv2d(conv6_stage1,
15,
1,
1,
activation_fn=None,
scope='conv7_stage1')
conv1_stage2 = layers.conv2d(image,
128,
9,
1,
activation_fn=None,
scope='conv1_stage2')
conv1_stage2 = tf.nn.relu(conv1_stage2)
pool1_stage2 = layers.max_pool2d(conv1_stage2, 3, 2)
conv2_stage2 = layers.conv2d(pool1_stage2,
128,
9,
1,
activation_fn=None,
scope='conv2_stage2')
conv2_stage2 = tf.nn.relu(conv2_stage2)
pool2_stage2 = layers.max_pool2d(conv2_stage2, 3, 2)
conv3_stage2 = layers.conv2d(pool2_stage2,
128,
9,
1,
activation_fn=None,
scope='conv3_stage2')
conv3_stage2 = tf.nn.relu(conv3_stage2)
pool3_stage2 = layers.max_pool2d(conv3_stage2, 3, 2)
conv4_stage2 = layers.conv2d(pool3_stage2,
32,
5,
1,
activation_fn=None,
scope='conv4_stage2')
conv4_stage2 = tf.nn.relu(conv4_stage2)
concat_stage2 = tf.concat(
axis=3, values=[conv4_stage2, conv7_stage1, pool_center_lower])
Mconv1_stage2 = layers.conv2d(concat_stage2,
128,
11,
1,
activation_fn=None,
scope='Mconv1_stage2')
Mconv1_stage2 = tf.nn.relu(Mconv1_stage2)
Mconv2_stage2 = layers.conv2d(Mconv1_stage2,
128,
11,
1,
activation_fn=None,
scope='Mconv2_stage2')
Mconv2_stage2 = tf.nn.relu(Mconv2_stage2)
Mconv3_stage2 = layers.conv2d(Mconv2_stage2,
128,
11,
1,
activation_fn=None,
scope='Mconv3_stage2')
Mconv3_stage2 = tf.nn.relu(Mconv3_stage2)
Mconv4_stage2 = layers.conv2d(Mconv3_stage2,
128,
1,
1,
activation_fn=None,
scope='Mconv4_stage2')
Mconv4_stage2 = tf.nn.relu(Mconv4_stage2)
Mconv5_stage2 = layers.conv2d(Mconv4_stage2,
15,
1,
1,
activation_fn=None,
scope='Mconv5_stage2')
conv1_stage3 = layers.conv2d(pool3_stage2,
32,
5,
1,
activation_fn=None,
scope='conv1_stage3')
conv1_stage3 = tf.nn.relu(conv1_stage3)
concat_stage3 = tf.concat(
axis=3, values=[conv1_stage3, Mconv5_stage2, pool_center_lower])
Mconv1_stage3 = layers.conv2d(concat_stage3,
128,
11,
1,
activation_fn=None,
scope='Mconv1_stage3')
Mconv1_stage3 = tf.nn.relu(Mconv1_stage3)
Mconv2_stage3 = layers.conv2d(Mconv1_stage3,
128,
11,
1,
activation_fn=None,
scope='Mconv2_stage3')
Mconv2_stage3 = tf.nn.relu(Mconv2_stage3)
Mconv3_stage3 = layers.conv2d(Mconv2_stage3,
128,
11,
1,
activation_fn=None,
scope='Mconv3_stage3')
Mconv3_stage3 = tf.nn.relu(Mconv3_stage3)
Mconv4_stage3 = layers.conv2d(Mconv3_stage3,
128,
1,
1,
activation_fn=None,
scope='Mconv4_stage3')
Mconv4_stage3 = tf.nn.relu(Mconv4_stage3)
Mconv5_stage3 = layers.conv2d(Mconv4_stage3,
15,
1,
1,
activation_fn=None,
scope='Mconv5_stage3')
conv1_stage4 = layers.conv2d(pool3_stage2,
32,
5,
1,
activation_fn=None,
scope='conv1_stage4')
conv1_stage4 = tf.nn.relu(conv1_stage4)
concat_stage4 = tf.concat(
axis=3, values=[conv1_stage4, Mconv5_stage3, pool_center_lower])
Mconv1_stage4 = layers.conv2d(concat_stage4,
128,
11,
1,
activation_fn=None,
scope='Mconv1_stage4')
Mconv1_stage4 = tf.nn.relu(Mconv1_stage4)
Mconv2_stage4 = layers.conv2d(Mconv1_stage4,
128,
11,
1,
activation_fn=None,
scope='Mconv2_stage4')
Mconv2_stage4 = tf.nn.relu(Mconv2_stage4)
Mconv3_stage4 = layers.conv2d(Mconv2_stage4,
128,
11,
1,
activation_fn=None,
scope='Mconv3_stage4')
Mconv3_stage4 = tf.nn.relu(Mconv3_stage4)
Mconv4_stage4 = layers.conv2d(Mconv3_stage4,
128,
1,
1,
activation_fn=None,
scope='Mconv4_stage4')
Mconv4_stage4 = tf.nn.relu(Mconv4_stage4)
Mconv5_stage4 = layers.conv2d(Mconv4_stage4,
15,
1,
1,
activation_fn=None,
scope='Mconv5_stage4')
conv1_stage5 = layers.conv2d(pool3_stage2,
32,
5,
1,
activation_fn=None,
scope='conv1_stage5')
conv1_stage5 = tf.nn.relu(conv1_stage5)
concat_stage5 = tf.concat(
axis=3, values=[conv1_stage5, Mconv5_stage4, pool_center_lower])
Mconv1_stage5 = layers.conv2d(concat_stage5,
128,
11,
1,
activation_fn=None,
scope='Mconv1_stage5')
Mconv1_stage5 = tf.nn.relu(Mconv1_stage5)
Mconv2_stage5 = layers.conv2d(Mconv1_stage5,
128,
11,
1,
activation_fn=None,
scope='Mconv2_stage5')
Mconv2_stage5 = tf.nn.relu(Mconv2_stage5)
Mconv3_stage5 = layers.conv2d(Mconv2_stage5,
128,
11,
1,
activation_fn=None,
scope='Mconv3_stage5')
Mconv3_stage5 = tf.nn.relu(Mconv3_stage5)
Mconv4_stage5 = layers.conv2d(Mconv3_stage5,
128,
1,
1,
activation_fn=None,
scope='Mconv4_stage5')
Mconv4_stage5 = tf.nn.relu(Mconv4_stage5)
Mconv5_stage5 = layers.conv2d(Mconv4_stage5,
15,
1,
1,
activation_fn=None,
scope='Mconv5_stage5')
conv1_stage6 = layers.conv2d(pool3_stage2,
32,
5,
1,
activation_fn=None,
scope='conv1_stage6')
conv1_stage6 = tf.nn.relu(conv1_stage6)
concat_stage6 = tf.concat(
axis=3, values=[conv1_stage6, Mconv5_stage5, pool_center_lower])
Mconv1_stage6 = layers.conv2d(concat_stage6,
128,
11,
1,
activation_fn=None,
scope='Mconv1_stage6')
Mconv1_stage6 = tf.nn.relu(Mconv1_stage6)
Mconv2_stage6 = layers.conv2d(Mconv1_stage6,
128,
11,
1,
activation_fn=None,
scope='Mconv2_stage6')
Mconv2_stage6 = tf.nn.relu(Mconv2_stage6)
Mconv3_stage6 = layers.conv2d(Mconv2_stage6,
128,
11,
1,
activation_fn=None,
scope='Mconv3_stage6')
Mconv3_stage6 = tf.nn.relu(Mconv3_stage6)
Mconv4_stage6 = layers.conv2d(Mconv3_stage6,
128,
1,
1,
activation_fn=None,
scope='Mconv4_stage6')
Mconv4_stage6 = tf.nn.relu(Mconv4_stage6)
Mconv5_stage6 = layers.conv2d(Mconv4_stage6,
15,
1,
1,
activation_fn=None,
scope='Mconv5_stage6')
return Mconv5_stage6
def cnn_model_hands(x_left, x_right, batch_size, is_train):
net_left = tf.reshape(x_left, [-1, 4, 39, 3])
net_right = tf.reshape(x_right, [-1, 4, 39, 3])
#add for spatial attention left
net_3_left = layers.convolution2d(net_left,
32, [3, 3],
padding='SAME',
scope='l_CNN_conv2_3')
net_3_left = layers.max_pool2d(net_3_left, [1, 3],
stride=[1, 2],
scope='l_CNN_maxpool2_3')
net_5_left = layers.convolution2d(net_left,
32, [3, 5],
padding='SAME',
scope='l_CNN_conv2_5')
net_5_left = layers.max_pool2d(net_5_left, [1, 3],
stride=[1, 2],
scope='l_CNN_maxpool2_5')
net_9_left = layers.convolution2d(net_left,
32, [3, 7],
padding='SAME',
scope='l_CNN_conv2_9')
net_9_left = layers.max_pool2d(net_9_left, [1, 3],
stride=[1, 2],
scope='l_CNN_maxpool2_9')
net_left = tf.concat([net_3_left, net_5_left, net_9_left], 3)
print(net_left.shape)
net_left = layers.convolution2d(net_left,
48, [3, 3],
padding='SAME',
scope='l_CNN_conv3')
net_left = layers.max_pool2d(net_left, [1, 3],
stride=2,
scope='l_CNN_maxpool3')
print(net_left.shape)
net_left = layers.flatten(net_left, scope='l_CNN_flatten')
net_left = layers.fully_connected(net_left,
512,
scope='l_CNN_fully_connect1')
net_left = layers.dropout(net_left,
keep_prob=0.5,
is_training=is_train,
scope='l_CNN_dropout')
#add for spatial attention right
net_3_right = layers.convolution2d(net_right,
32, [3, 3],
padding='SAME',
scope='r_CNN_conv2_3')
net_3_right = layers.max_pool2d(net_3_right, [1, 3],
stride=[1, 2],
scope='r_CNN_maxpool2_3')
net_5_right = layers.convolution2d(net_right,
32, [3, 5],
padding='SAME',
scope='r_CNN_conv2_5')
net_5_right = layers.max_pool2d(net_5_right, [1, 3],
stride=[1, 2],
scope='r_CNN_maxpool2_5')
net_9_right = layers.convolution2d(net_right,
32, [3, 7],
padding='SAME',
scope='r_CNN_conv2_9')
net_9_right = layers.max_pool2d(net_9_right, [1, 3],
stride=[1, 2],
scope='r_CNN_maxpool2_9')
net_right = tf.concat([net_3_right, net_5_right, net_9_right], 3)
print(net_right.shape)
net_right = layers.convolution2d(net_right,
48, [3, 3],
padding='SAME',
scope='r_CNN_conv3')
net_right = layers.max_pool2d(net_right, [1, 3],
stride=2,
scope='r_CNN_maxpool3')
print(net_right.shape)
net_right = layers.flatten(net_right, scope='r_CNN_flatten')
net_right = layers.fully_connected(net_right,
512,
scope='r_CNN_fully_connect1')
net_right = layers.dropout(net_right,
keep_prob=0.5,
is_training=is_train,
scope='r_CNN_dropout')
# independant classifier for two hands
net_hands = tf.concat([net_left, net_right], 1)
net_hands = layers.flatten(net_hands, scope='lr_CNN_flatten')
net_hands = layers.fully_connected(net_hands,
256,
scope='lr_CNN_fully_connect1')
net_hands = layers.dropout(net_hands,
keep_prob=0.5,
is_training=is_train,
scope='lr_CNN_dropout')
net_hands = layers.fully_connected(net_hands,
20,
scope='lr_CNN_fully_connect2')
return net_hands, net_left, net_right
def create_model(self, input, options=None):
# STEM Network
with tf.variable_scope('stem'):
self.conv2d_1 = conv2d(input, num_outputs=64,
kernel_size=[7,7], stride=2, padding="SAME",
activation_fn=tf.nn.relu)
self.pool_1 = max_pool2d(self.conv2d_1, kernel_size=[3,3], stride=2, padding='SAME')
self.lrn_1 = tf.nn.local_response_normalization(self.pool_1, bias=2, alpha=0.0001,beta=0.75)
self.conv2d_2 = conv2d(self.lrn_1, num_outputs=64,
kernel_size=[1,1], stride=1, padding="SAME",
activation_fn=tf.nn.relu)
self.conv2d_3 = conv2d(self.conv2d_2, num_outputs=192,
kernel_size=[3,3], stride=1, padding="SAME",
activation_fn=tf.nn.relu)
self.lrn_2 = tf.nn.local_response_normalization(self.conv2d_3, bias=2, alpha=0.0001,beta=0.75)
self.pool_2 = max_pool2d(self.lrn_2, kernel_size=[3,3], stride=2, padding='SAME')
# Inception 3
# a, b
inception_3_nums = {
'conv2d_1' : [64 , 128],
'conv2d_2_1': [96 , 128],
'conv2d_2_2': [128, 192],
'conv2d_3_1': [16 , 32],
'conv2d_3_2': [32 , 96],
'conv2d_4' : [32 , 64]
}
with tf.variable_scope('inception_3'):
prev = self.pool_2
for i in range(2):
conv2d_1_kernels = inception_3_nums['conv2d_1'][i]
conv2d_2_1_kernels = inception_3_nums['conv2d_2_1'][i]
conv2d_2_2_kernels = inception_3_nums['conv2d_2_2'][i]
conv2d_3_1_kernels = inception_3_nums['conv2d_3_1'][i]
conv2d_3_2_kernels = inception_3_nums['conv2d_3_2'][i]
conv2d_4_kernels = inception_3_nums['conv2d_4'][i]
conv2d_1 = conv2d(prev, num_outputs=conv2d_1_kernels,
kernel_size=[1,1], stride=1, padding="SAME",
activation_fn=tf.nn.relu)
conv2d_2 = conv2d(prev, num_outputs=conv2d_2_1_kernels,
kernel_size=[1,1], stride=1, padding="SAME",
activation_fn=tf.nn.relu)
conv2d_2 = conv2d(conv2d_2, num_outputs=conv2d_2_2_kernels,
kernel_size=[3,3], stride=1, padding="SAME",
activation_fn=tf.nn.relu)
conv2d_3 = conv2d(prev, num_outputs=conv2d_3_1_kernels,
kernel_size=[1,1], stride=1, padding="SAME",
activation_fn=tf.nn.relu)
conv2d_3 = conv2d(conv2d_3, num_outputs=conv2d_3_2_kernels,
kernel_size=[5,5], stride=1, padding="SAME",
activation_fn=tf.nn.relu)
conv2d_4 = max_pool2d(prev, kernel_size=[3,3], stride=1, padding='SAME')
conv2d_4 = conv2d(conv2d_4, num_outputs=conv2d_4_kernels,
kernel_size=[1,1], stride=1, padding="SAME",
activation_fn=tf.nn.relu)
layers_concat = list()
layers_concat.append(conv2d_1)
layers_concat.append(conv2d_2)
layers_concat.append(conv2d_3)
layers_concat.append(conv2d_4)
prev = tf.concat(layers_concat, 3)
if i is 0:
self.inception_3a = prev
prev = max_pool2d(prev, kernel_size=[3,3], stride=2, padding='SAME')
self.inception_3b = prev
# Inception (4)
# a, b, c, d, e
inception_4_nums = {
'conv2d_1' : [192, 160, 128, 112, 256],
'conv2d_2_1': [96 , 112, 128, 144, 160],
'conv2d_2_2': [208, 224, 256, 228, 320],
'conv2d_3_1': [16 , 24, 24, 32, 32],
'conv2d_3_2': [48 , 64, 64, 64, 128],
'conv2d_4' : [64 , 64, 64, 64, 128]
}
with tf.variable_scope('inception_4'):
for i in range(5):
conv2d_1_kernels = inception_4_nums['conv2d_1'][i]
conv2d_2_1_kernels = inception_4_nums['conv2d_2_1'][i]
conv2d_2_2_kernels = inception_4_nums['conv2d_2_2'][i]
conv2d_3_1_kernels = inception_4_nums['conv2d_3_1'][i]
conv2d_3_2_kernels = inception_4_nums['conv2d_3_2'][i]
conv2d_4_kernels = inception_4_nums['conv2d_4'][i]
conv2d_1 = conv2d(prev, num_outputs=conv2d_1_kernels,
kernel_size=[1,1], stride=1, padding="SAME",
activation_fn=tf.nn.relu)
conv2d_2 = conv2d(prev, num_outputs=conv2d_2_1_kernels,
kernel_size=[1,1], stride=1, padding="SAME",
activation_fn=tf.nn.relu)
conv2d_2 = conv2d(conv2d_2, num_outputs=conv2d_2_2_kernels,
kernel_size=[3,3], stride=1, padding="SAME",
activation_fn=tf.nn.relu)
conv2d_3 = conv2d(prev, num_outputs=conv2d_3_1_kernels,
kernel_size=[1,1], stride=1, padding="SAME",
activation_fn=tf.nn.relu)
conv2d_3 = conv2d(conv2d_3, num_outputs=conv2d_3_2_kernels,
kernel_size=[5,5], stride=1, padding="SAME",
activation_fn=tf.nn.relu)
conv2d_4 = max_pool2d(prev, kernel_size=[3,3], stride=1, padding='SAME')
conv2d_4 = conv2d(conv2d_4, num_outputs=conv2d_4_kernels,
kernel_size=[1,1], stride=1, padding="SAME",
activation_fn=tf.nn.relu)
layers_concat = list()
layers_concat.append(conv2d_1)
layers_concat.append(conv2d_2)
layers_concat.append(conv2d_3)
layers_concat.append(conv2d_4)
prev = tf.concat(layers_concat, 3)
if i is 0:
self.inception_4a = prev
elif i is 1:
self.inception_4b = prev
elif i is 2:
self.inception_4c = prev
elif i is 3:
self.inception_4d = prev
prev = max_pool2d(prev, kernel_size=[3,3], stride=2, padding='SAME')
self.inception_4e = prev
# Inception (5)
# a, b
inception_5_nums = {
'conv2d_1' : [256, 384],
'conv2d_2_1': [160, 192],
'conv2d_2_2': [320, 384],
'conv2d_3_1': [32 , 48],
'conv2d_3_2': [128, 128],
'conv2d_4' : [128, 128]
}
with tf.variable_scope('inception_5'):
for i in range(2):
conv2d_1_kernels = inception_5_nums['conv2d_1'][i]
conv2d_2_1_kernels = inception_5_nums['conv2d_2_1'][i]
conv2d_2_2_kernels = inception_5_nums['conv2d_2_2'][i]
conv2d_3_1_kernels = inception_5_nums['conv2d_3_1'][i]
conv2d_3_2_kernels = inception_5_nums['conv2d_3_2'][i]
conv2d_4_kernels = inception_5_nums['conv2d_4'][i]
conv2d_1 = conv2d(prev, num_outputs=conv2d_1_kernels,
kernel_size=[1,1], stride=1, padding="SAME",
activation_fn=tf.nn.relu)
conv2d_2 = conv2d(prev, num_outputs=conv2d_2_1_kernels,
kernel_size=[1,1], stride=1, padding="SAME",
activation_fn=tf.nn.relu)
conv2d_2 = conv2d(conv2d_2, num_outputs=conv2d_2_2_kernels,
kernel_size=[3,3], stride=1, padding="SAME",
activation_fn=tf.nn.relu)
conv2d_3 = conv2d(prev, num_outputs=conv2d_3_1_kernels,
kernel_size=[1,1], stride=1, padding="SAME",
activation_fn=tf.nn.relu)
conv2d_3 = conv2d(conv2d_3, num_outputs=conv2d_3_2_kernels,
kernel_size=[5,5], stride=1, padding="SAME",
activation_fn=tf.nn.relu)
conv2d_4 = max_pool2d(prev, kernel_size=[3,3], stride=1, padding='SAME')
conv2d_4 = conv2d(conv2d_4, num_outputs=conv2d_4_kernels,
kernel_size=[1,1], stride=1, padding="SAME",
activation_fn=tf.nn.relu)
layers_concat = list()
layers_concat.append(conv2d_1)
layers_concat.append(conv2d_2)
layers_concat.append(conv2d_3)
layers_concat.append(conv2d_4)
prev = tf.concat(layers_concat, 3)
if i is 0:
self.inception_5a = prev
self.inception_5b = prev
with tf.variable_scope('final'):
# Aux #1 output
aux_avg_pool_1 = avg_pool2d(self.inception_4a, kernel_size=[5,5], stride=3, padding='SAME')
aux_conv2d_1 = conv2d(aux_avg_pool_1, num_outputs=128,
kernel_size=[1,1], stride=1, padding="SAME",
activation_fn=tf.nn.relu)
aux_flat = flatten(aux_conv2d_1)
aux_fcl_1 = fully_connected(aux_flat, num_outputs=1024, activation_fn=tf.nn.relu)
aux_droupout_1 = tf.nn.dropout(aux_fcl_1, 0.7)
self.aux_1_out = fully_connected(aux_droupout_1, num_outputs=1024, activation_fn=tf.nn.relu)
# Aux #2 output
aux_avg_pool_1 = avg_pool2d(self.inception_4d, kernel_size=[5,5], stride=3, padding='SAME')
aux_conv2d_1 = conv2d(aux_avg_pool_1, num_outputs=128,
kernel_size=[1,1], stride=1, padding="SAME",
activation_fn=tf.nn.relu)
aux_flat = flatten(aux_conv2d_1)
aux_fcl_1 = fully_connected(aux_flat, num_outputs=1024, activation_fn=tf.nn.relu)
aux_droupout_1 = tf.nn.dropout(aux_fcl_1, 0.7)
self.aux_2_out = fully_connected(aux_droupout_1, num_outputs=1024, activation_fn=tf.nn.relu)
# Final output
self.final_avg_pool_1 = avg_pool2d(prev, kernel_size=[7,7], stride=1, padding='SAME')
self.final_dropout = tf.nn.dropout(self.final_avg_pool_1, 0.4)
self.final_flat = flatten(self.final_dropout)
self.final_out = fully_connected(self.final_flat, num_outputs=self.num_classes, activation_fn=None)
return [self.aux_1_out, self.aux_2_out, self.final_out]
def _build_net(self, inputs, goal_pos, RNN_SIZE, TRAINING, a_size):
w_init = layers.variance_scaling_initializer()
conv1 = layers.conv2d(inputs=inputs,
padding="SAME",
num_outputs=RNN_SIZE // 4,
kernel_size=[3, 3],
stride=1,
data_format="NHWC",
weights_initializer=w_init,
activation_fn=tf.nn.relu)
conv1a = layers.conv2d(inputs=conv1,
padding="SAME",
num_outputs=RNN_SIZE // 4,
kernel_size=[3, 3],
stride=1,
data_format="NHWC",
weights_initializer=w_init,
activation_fn=tf.nn.relu)
conv1b = layers.conv2d(inputs=conv1a,
padding="SAME",
num_outputs=RNN_SIZE // 4,
kernel_size=[3, 3],
stride=1,
data_format="NHWC",
weights_initializer=w_init,
activation_fn=tf.nn.relu)
pool1 = layers.max_pool2d(inputs=conv1b, kernel_size=[2, 2])
conv2 = layers.conv2d(inputs=pool1,
padding="SAME",
num_outputs=RNN_SIZE // 2,
kernel_size=[3, 3],
stride=1,
data_format="NHWC",
weights_initializer=w_init,
activation_fn=tf.nn.relu)
conv2a = layers.conv2d(inputs=conv2,
padding="SAME",
num_outputs=RNN_SIZE // 2,
kernel_size=[3, 3],
stride=1,
data_format="NHWC",
weights_initializer=w_init,
activation_fn=tf.nn.relu)
conv2b = layers.conv2d(inputs=conv2a,
padding="SAME",
num_outputs=RNN_SIZE // 2,
kernel_size=[3, 3],
stride=1,
data_format="NHWC",
weights_initializer=w_init,
activation_fn=tf.nn.relu)
pool2 = layers.max_pool2d(inputs=conv2b, kernel_size=[2, 2])
conv3 = layers.conv2d(inputs=pool2,
padding="VALID",
num_outputs=RNN_SIZE - GOAL_REPR_SIZE,
kernel_size=[2, 2],
stride=1,
data_format="NHWC",
weights_initializer=w_init,
activation_fn=None)
flat = tf.nn.relu(layers.flatten(conv3))
goal_layer = layers.fully_connected(inputs=goal_pos,
num_outputs=GOAL_REPR_SIZE)
hidden_input = tf.concat([flat, goal_layer], 1)
h1 = layers.fully_connected(inputs=hidden_input, num_outputs=RNN_SIZE)
d1 = layers.dropout(h1, keep_prob=KEEP_PROB1, is_training=TRAINING)
h2 = layers.fully_connected(inputs=d1,
num_outputs=RNN_SIZE,
activation_fn=None)
d2 = layers.dropout(h2, keep_prob=KEEP_PROB2, is_training=TRAINING)
self.h3 = tf.nn.relu(d2 + hidden_input)
#Recurrent network for temporal dependencies
lstm_cell = tf.nn.rnn_cell.BasicLSTMCell(RNN_SIZE, state_is_tuple=True)
c_init = np.zeros((1, lstm_cell.state_size.c), np.float32)
h_init = np.zeros((1, lstm_cell.state_size.h), np.float32)
state_init = [c_init, h_init]
c_in = tf.placeholder(tf.float32, [1, lstm_cell.state_size.c])
h_in = tf.placeholder(tf.float32, [1, lstm_cell.state_size.h])
state_in = (c_in, h_in)
rnn_in = tf.expand_dims(self.h3, [0])
step_size = tf.shape(inputs)[:1]
state_in = tf.nn.rnn_cell.LSTMStateTuple(c_in, h_in)
lstm_outputs, lstm_state = tf.nn.dynamic_rnn(lstm_cell,
rnn_in,
initial_state=state_in,
sequence_length=step_size,
time_major=False)
lstm_c, lstm_h = lstm_state
state_out = (lstm_c[:1, :], lstm_h[:1, :])
self.rnn_out = tf.reshape(lstm_outputs, [-1, RNN_SIZE])
policy_layer = layers.fully_connected(
inputs=self.rnn_out,
num_outputs=a_size,
weights_initializer=normalized_columns_initializer(1. /
float(a_size)),
biases_initializer=None,
activation_fn=None)
policy = tf.nn.softmax(policy_layer)
policy_sig = tf.sigmoid(policy_layer)
value = layers.fully_connected(
inputs=self.rnn_out,
num_outputs=1,
weights_initializer=normalized_columns_initializer(1.0),
biases_initializer=None,
activation_fn=None)
blocking = layers.fully_connected(
inputs=self.rnn_out,
num_outputs=1,
weights_initializer=normalized_columns_initializer(1.0),
biases_initializer=None,
activation_fn=tf.sigmoid)
on_goal = layers.fully_connected(
inputs=self.rnn_out,
num_outputs=1,
weights_initializer=normalized_columns_initializer(1.0),
biases_initializer=None,
activation_fn=tf.sigmoid)
return policy, value, state_out, state_in, state_init, blocking, on_goal, policy_sig
def build(self, inputs, batch_size, sequence_length,
spatial_fully_connected_size, learning_rate, weight_decay,
is_training, reuse_weights):
bn_params = {
'decay': 0.999,
'center': True,
'scale': True,
'epsilon': 0.001,
'updates_collections': None,
'is_training': is_training,
}
input_shape = inputs.get_shape()
horizontal_slice_size = int(round(int(input_shape[2]) / 3))
vertical_slice_size = int(round(int(input_shape[1]) / 3))
inputs = tf.slice(inputs,
begin=[0, vertical_slice_size, 0, 0],
size=[-1, -1, horizontal_slice_size * 2, -1])
self.final_gradient = tf.placeholder(
tf.float32,
shape=(sequence_length, spatial_fully_connected_size),
name='x___final_gradient_ph')
with tf.contrib.framework.arg_scope(
[layers.convolution2d],
kernel_size=3,
stride=1,
padding='SAME',
rate=1,
activation_fn=tf.nn.relu,
normalizer_fn=None,
weights_initializer=None,
weights_regularizer=layers.l2_regularizer(weight_decay)):
net = layers.convolution2d(inputs,
64,
scope='conv1_1',
reuse=reuse_weights)
net = layers.convolution2d(net,
64,
scope='conv1_2',
reuse=reuse_weights)
net = layers.max_pool2d(net, 2, 2, scope='pool1')
net = layers.convolution2d(net,
128,
scope='conv2_1',
reuse=reuse_weights)
net = layers.convolution2d(net,
128,
scope='conv2_2',
reuse=reuse_weights)
net = layers.max_pool2d(net, 2, 2, scope='pool2')
net = layers.convolution2d(net,
256,
scope='conv3_1',
reuse=reuse_weights)
net = layers.convolution2d(net,
256,
scope='conv3_2',
reuse=reuse_weights)
net = layers.convolution2d(net,
256,
scope='conv3_3',
reuse=reuse_weights)
net = layers.max_pool2d(net, 2, 2, scope='pool3')
net = layers.convolution2d(net,
512,
scope='conv4_1',
reuse=reuse_weights)
net = layers.convolution2d(net,
512,
scope='conv4_2',
reuse=reuse_weights)
net = layers.convolution2d(net,
512,
scope='conv4_3',
reuse=reuse_weights)
net = layers.max_pool2d(net, 2, 2, scope='pool4')
net = layers.convolution2d(net,
512,
scope='conv5_1',
reuse=reuse_weights)
net = layers.convolution2d(net,
512,
scope='conv5_2',
reuse=reuse_weights)
net = layers.convolution2d(net,
512,
scope='conv5_3',
reuse=reuse_weights)
net = layers.max_pool2d(net, 2, 2, scope='pool5')
net = layers.flatten(net)
with tf.contrib.framework.arg_scope(
[layers.fully_connected],
activation_fn=tf.nn.relu,
normalizer_fn=layers.batch_norm,
normalizer_params=bn_params,
weights_initializer=layers.variance_scaling_initializer(),
weights_regularizer=layers.l2_regularizer(weight_decay)):
net = layers.fully_connected(net,
spatial_fully_connected_size,
scope='spatial_FC',
reuse=reuse_weights)
self.representation = layers.flatten(net)
self.loss = tf.matmul(self.representation,
tf.transpose(self.final_gradient),
name='x___spatial_loss')
self.partial_run_setup_objs = [self.representation, self.loss]
if is_training:
self.trainer = tf.train.AdamOptimizer(learning_rate)
self.train_op = self.trainer.minimize(self.loss)
with tf.control_dependencies([self.train_op]):
self.with_train_op = self.loss
self.partial_run_setup_objs.append(self.train_op)
def inference_u_net(images: tf.Tensor,
params: ModelParams,
num_classes: int,
use_batch_norm=False,
weight_decay=0.0,
is_training=False) -> tf.Tensor:
enc_layers = OrderedDict()
dec_layers = OrderedDict()
with tf.variable_scope('U-Net'):
with tf.variable_scope('Encoder'):
conv_layer = layers.conv2d(images,
num_outputs=64,
kernel_size=(3, 3),
padding='SAME',
activation_fn=tf.identity)
enc_layers['conv_layer_enc_64'] = conv_bn_layer(
conv_layer,
kernel_size=(3, 3),
output_channels=64,
bn=True,
is_training=is_training,
relu=True)
conv_layer = layers.max_pool2d(
inputs=enc_layers['conv_layer_enc_64'],
kernel_size=(2, 2),
stride=2)
for n_feat in [128, 256, 512]:
enc_layers['conv_layer_enc_' + str(n_feat)] = conv_bn_layer(
conv_layer,
kernel_size=(3, 3),
output_channels=n_feat,
bn=True,
is_training=is_training,
relu=True)
enc_layers['conv_layer_enc_' + str(n_feat)] = conv_bn_layer(
enc_layers['conv_layer_enc_' + str(n_feat)],
kernel_size=(3, 3),
output_channels=n_feat,
bn=True,
is_training=is_training,
relu=True)
conv_layer = layers.max_pool2d(
inputs=enc_layers['conv_layer_enc_' + str(n_feat)],
kernel_size=(2, 2),
stride=2)
conv_layer_enc_1024 = conv_bn_layer(conv_layer,
kernel_size=(3, 3),
output_channels=1024,
bn=True,
is_training=is_training,
relu=True)
with tf.variable_scope('Decoder'):
dec_layers['conv_layer_dec_512'] = conv_bn_layer(
conv_layer_enc_1024,
kernel_size=(3, 3),
output_channels=512,
bn=True,
is_training=is_training,
relu=True)
reduced_patchsize = _get_image_shape_tensor(
enc_layers['conv_layer_enc_512'])
dec_layers['conv_layer_dec_512'] = tf.image.resize_images(
dec_layers['conv_layer_dec_512'],
size=reduced_patchsize,
method=tf.image.ResizeMethod.BILINEAR)
for n_feat in [512, 256, 128, 64]:
dec_layers['conv_layer_dec_' + str(n_feat * 2)] = tf.concat(
[
dec_layers['conv_layer_dec_' + str(n_feat)],
enc_layers['conv_layer_enc_' + str(n_feat)]
],
axis=3)
dec_layers['conv_layer_dec_' + str(n_feat)] = conv_bn_layer(
dec_layers['conv_layer_dec_' + str(n_feat * 2)],
kernel_size=(3, 3),
output_channels=n_feat,
bn=True,
is_training=is_training,
relu=True)
if n_feat > 64:
dec_layers['conv_layer_dec_' +
str(int(n_feat / 2))] = conv_bn_layer(
dec_layers['conv_layer_dec_' + str(n_feat)],
kernel_size=(3, 3),
output_channels=n_feat / 2,
bn=True,
is_training=is_training,
relu=True)
reduced_patchsize = _get_image_shape_tensor(
enc_layers['conv_layer_enc_' + str(int(n_feat / 2))])
dec_layers['conv_layer_dec_' +
str(int(n_feat / 2))] = tf.image.resize_images(
dec_layers['conv_layer_dec_' +
str(int(n_feat / 2))],
size=reduced_patchsize,
method=tf.image.ResizeMethod.BILINEAR)
return layers.conv2d(dec_layers['conv_layer_dec_64'],
num_outputs=num_classes,
kernel_size=(3, 3),
padding='SAME',
activation_fn=tf.identity)
def build(self, spatial_fully_connected_size,
temporal_fully_connected_layers, inputs, labels, weight_decay,
vgg_init_dir, is_training):
bn_params = {
'decay': 0.999,
'center': True,
'scale': True,
'epsilon': 0.001,
'updates_collections': None,
'is_training': is_training,
}
if is_training:
vgg_layers, vgg_layer_names = read_vgg_init(vgg_init_dir)
batch_size = tf.shape(inputs)[0]
inputs_shape = inputs.get_shape()
horizontal_slice_size = int(round(int(inputs_shape[3]) / 3))
vertical_slice_size = int(round(int(inputs_shape[2]) / 3))
inputs = tf.slice(inputs,
begin=[0, 0, vertical_slice_size, 0, 0],
size=[-1, -1, -1, horizontal_slice_size * 2, -1])
concated = None
reuse = None
for sequence_image in range(int(inputs_shape[1])):
with tf.contrib.framework.arg_scope(
[layers.convolution2d],
kernel_size=3,
stride=1,
padding='SAME',
rate=1,
activation_fn=tf.nn.relu,
normalizer_fn=None,
weights_initializer=None,
weights_regularizer=layers.l2_regularizer(weight_decay)):
net = layers.convolution2d(inputs[:, sequence_image],
64,
scope='conv1_1',
reuse=reuse)
net = layers.convolution2d(net,
64,
scope='conv1_2',
reuse=reuse)
net = layers.max_pool2d(net, 2, 2, scope='pool1')
net = layers.convolution2d(net,
128,
scope='conv2_1',
reuse=reuse)
net = layers.convolution2d(net,
128,
scope='conv2_2',
reuse=reuse)
net = layers.max_pool2d(net, 2, 2, scope='pool2')
net = layers.convolution2d(net,
256,
scope='conv3_1',
reuse=reuse)
net = layers.convolution2d(net,
256,
scope='conv3_2',
reuse=reuse)
net = layers.convolution2d(net,
256,
scope='conv3_3',
reuse=reuse)
net = layers.max_pool2d(net, 2, 2, scope='pool3')
net = layers.convolution2d(net,
512,
scope='conv4_1',
reuse=reuse)
net = layers.convolution2d(net,
512,
scope='conv4_2',
reuse=reuse)
net = layers.convolution2d(net,
512,
scope='conv4_3',
reuse=reuse)
net = layers.max_pool2d(net, 2, 2, scope='pool4')
net = layers.convolution2d(net,
512,
scope='conv5_1',
reuse=reuse)
net = layers.convolution2d(net,
512,
scope='conv5_2',
reuse=reuse)
net = layers.convolution2d(net,
512,
scope='conv5_3',
reuse=reuse)
net = layers.max_pool2d(net, 2, 2, scope='pool5')
net_shape = net.get_shape()
net = tf.reshape(net, [
batch_size,
int(net_shape[1]) * int(net_shape[2]) * int(net_shape[3])
])
with tf.contrib.framework.arg_scope(
[layers.fully_connected],
activation_fn=tf.nn.relu,
normalizer_fn=layers.batch_norm,
normalizer_params=bn_params,
weights_initializer=layers.variance_scaling_initializer(),
weights_regularizer=layers.l2_regularizer(weight_decay)):
net = layers.fully_connected(net,
spatial_fully_connected_size,
scope='spatial_FC',
reuse=reuse)
if concated is None:
concated = tf.expand_dims(net, axis=1)
else:
concated = tf.concat(
[concated, tf.expand_dims(net, axis=1)], axis=1)
reuse = True
if is_training:
init_op, init_feed, pretrained_vars = create_init_op(vgg_layers)
self.pretrained_vars = pretrained_vars
net = concated
net_shape = net.get_shape()
net = tf.reshape(
net,
[batch_size, int(net_shape[1]) * int(net_shape[2])])
with tf.contrib.framework.arg_scope(
[layers.fully_connected],
activation_fn=tf.nn.relu,
normalizer_fn=layers.batch_norm,
normalizer_params=bn_params,
weights_initializer=layers.variance_scaling_initializer(),
weights_regularizer=layers.l2_regularizer(weight_decay)):
layer_num = 1
for fully_connected_num in temporal_fully_connected_layers:
net = layers.fully_connected(
net,
fully_connected_num,
scope='temporal_FC{}'.format(layer_num))
layer_num += 1
logits = layers.fully_connected(
net,
2,
activation_fn=None,
weights_initializer=layers.xavier_initializer(),
weights_regularizer=layers.l2_regularizer(weight_decay),
biases_initializer=tf.zeros_initializer(),
scope='logits')
total_loss = loss(logits, labels, is_training)
if is_training:
return logits, total_loss, init_op, init_feed
else:
return logits, total_loss
def googlenet(inputs,
dropout_keep_prob=0.4,
num_classes=1000,
is_training=True,
restore_logits=None,
scope=''):
'''
Implementation of https://arxiv.org/pdf/1409.4842.pdf
'''
end_points = {}
with tf.name_scope(scope, "googlenet", [inputs]):
with ops.arg_scope([layers.max_pool2d], padding='SAME'):
end_points['conv0'] = layers.conv2d(inputs,
64, [7, 7],
stride=2,
scope='conv0')
end_points['pool0'] = layers.max_pool2d(end_points['conv0'],
[3, 3],
scope='pool0')
end_points['conv1_a'] = layers.conv2d(end_points['pool0'],
64, [1, 1],
scope='conv1_a')
end_points['conv1_b'] = layers.conv2d(end_points['conv1_a'],
192, [3, 3],
scope='conv1_b')
end_points['pool1'] = layers.max_pool2d(end_points['conv1_b'],
[3, 3],
scope='pool1')
with tf.variable_scope("inception_3a"):
end_points['inception_3a'] = get_inception_layer(
end_points['pool1'], 64, 96, 128, 16, 32, 32)
with tf.variable_scope("inception_3b"):
end_points['inception_3b'] = get_inception_layer(
end_points['inception_3a'], 128, 128, 192, 32, 96, 64)
end_points['pool2'] = layers.max_pool2d(end_points['inception_3b'],
[3, 3],
scope='pool2')
with tf.variable_scope("inception_4a"):
end_points['inception_4a'] = get_inception_layer(
end_points['pool2'], 192, 96, 208, 16, 48, 64)
with tf.variable_scope("aux_logits_1"):
end_points['aux_logits_1'] = aux_logit_layer(
end_points['inception_4a'], num_classes, is_training)
with tf.variable_scope("inception_4b"):
end_points['inception_4b'] = get_inception_layer(
end_points['inception_4a'], 160, 112, 224, 24, 64, 64)
with tf.variable_scope("inception_4c"):
end_points['inception_4c'] = get_inception_layer(
end_points['inception_4b'], 128, 128, 256, 24, 64, 64)
with tf.variable_scope("inception_4d"):
end_points['inception_4d'] = get_inception_layer(
end_points['inception_4c'], 112, 144, 288, 32, 64, 64)
with tf.variable_scope("aux_logits_2"):
end_points['aux_logits_2'] = aux_logit_layer(
end_points['inception_4d'], num_classes, is_training)
with tf.variable_scope("inception_4e"):
end_points['inception_4e'] = get_inception_layer(
end_points['inception_4d'], 256, 160, 320, 32, 128, 128)
end_points['pool3'] = layers.max_pool2d(end_points['inception_4e'],
[3, 3],
scope='pool3')
with tf.variable_scope("inception_5a"):
end_points['inception_5a'] = get_inception_layer(
end_points['pool3'], 256, 160, 320, 32, 128, 128)
with tf.variable_scope("inception_5b"):
end_points['inception_5b'] = get_inception_layer(
end_points['inception_5a'], 384, 192, 384, 48, 128, 128)
end_points['pool4'] = layers.avg_pool2d(end_points['inception_5b'],
[7, 7],
stride=1,
scope='pool4')
end_points['reshape'] = tf.reshape(end_points['pool4'], [-1, 1024])
end_points['dropout'] = layers.dropout(end_points['reshape'],
dropout_keep_prob,
is_training=is_training)
end_points['logits'] = layers.fully_connected(
end_points['dropout'],
num_classes,
activation_fn=None,
scope='logits')
end_points['predictions'] = tf.nn.softmax(end_points['logits'],
name='predictions')
return end_points['logits']
def __init__(self, word_embeddings, setting):
self.vocab_size = setting.vocab_size
self.len_sentence= len_sentence = setting.len_sentence
self.num_epochs = setting.num_epochs
self.num_classes = num_classes =setting.num_classes
self.cnn_size = setting.cnn_size
self.num_layers = setting.num_layers
self.pos_size = setting.pos_size
self.pos_num = setting.pos_num
self.word_embedding = setting.word_embedding
self.lr = setting.lr
word_embedding = tf.get_variable(initializer=word_embeddings, name='word_embedding')
pos1_embedding = tf.get_variable('pos1_embedding', [self.pos_num, self.pos_size])
pos2_embedding = tf.get_variable('pos2_embedding', [self.pos_num, self.pos_size])
#relation_embedding = tf.get_variable('relation_embedding', [self.num_classes, self.cnn_size])
self.input_word = tf.placeholder(dtype=tf.int32, shape=[None, len_sentence], name='input_word')
self.input_pos1 = tf.placeholder(dtype=tf.int32, shape=[None, len_sentence], name='input_pos1')
self.input_pos2 = tf.placeholder(dtype=tf.int32, shape=[None, len_sentence], name='input_pos2')
self.input_y = tf.placeholder(dtype=tf.float32, shape=[None, num_classes], name='input_y')
self.keep_prob = tf.placeholder(tf.float32)
self.input_word_ebd = tf.nn.embedding_lookup(word_embedding, self.input_word)
self.input_pos1_ebd = tf.nn.embedding_lookup(pos1_embedding, self.input_pos1)
self.input_pos2_ebd = tf.nn.embedding_lookup(pos2_embedding, self.input_pos2)
self.inputs = tf.concat(axis=2,values=[self.input_word_ebd,self.input_pos1_ebd,self.input_pos2_ebd])
self.inputs = tf.reshape(self.inputs, [-1,self.len_sentence,self.word_embedding+self.pos_size*2,1] )
conv = layers.conv2d(inputs =self.inputs ,num_outputs = self.cnn_size ,kernel_size = [3,60],stride=[1,60],padding='SAME')
max_pool = layers.max_pool2d(conv,kernel_size = [70,1],stride=[1,1])
self.sentence = tf.reshape(max_pool, [-1, self.cnn_size])
tanh = tf.nn.tanh(self.sentence)
drop = layers.dropout(tanh,keep_prob=self.keep_prob)
self.outputs = layers.fully_connected(inputs = drop,num_outputs = self.num_classes,activation_fn = tf.nn.softmax)
'''
self.y_index = tf.argmax(self.input_y,1,output_type=tf.int32)
self.indexes = tf.range(0, tf.shape(self.outputs)[0]) * tf.shape(self.outputs)[1] + self.y_index
self.responsible_outputs = - tf.reduce_mean(tf.log(tf.gather(tf.reshape(self.outputs, [-1]),self.indexes)))
'''
#loss
self.cross_loss = -tf.reduce_mean( tf.log(tf.reduce_sum( self.input_y * self.outputs ,axis=1)))
self.reward = tf.log(tf.reduce_sum( self.input_y * self.outputs ,axis=1))
self.l2_loss = tf.contrib.layers.apply_regularization(regularizer=tf.contrib.layers.l2_regularizer(0.0001),
weights_list=tf.trainable_variables())
self.final_loss = self.cross_loss + self.l2_loss
#accuracy
self.pred = tf.argmax(self.outputs,axis=1)
self.pred_prob = tf.reduce_max(self.outputs,axis=1)
self.y_label = tf.argmax(self.input_y,axis=1)
self.accuracy = tf.reduce_mean(tf.cast( tf.equal(self.pred,self.y_label), 'float'))
#minimize loss
optimizer = tf.train.AdamOptimizer(self.lr)
self.train_op = optimizer.minimize(self.final_loss)
self.tvars = tf.trainable_variables()
# manual update parameters
self.tvars_holders = []
for idx, var in enumerate(self.tvars):
placeholder = tf.placeholder(tf.float32, name=str(idx) + '_holder')
self.tvars_holders.append(placeholder)
self.update_tvar_holder = []
for idx, var in enumerate(self.tvars):
update_tvar = tf.assign(var, self.tvars_holders[idx])
self.update_tvar_holder.append(update_tvar)
def generator_conv(self, image, reuse=False):
with tf.variable_scope('compressor') as scope:
if reuse:
scope.reuse_variables()
'''
conv1 = ly.conv2d(image,96,kernel_size=11,stride=4,padding='SAME',activation_fn=tf.nn.relu,weights_initializer=tf.random_normal_initializer(0, 0.02))
conv1 = self.bo_batch_norm(conv1, self.is_train)
conv1 = ly.max_pool2d(conv1,kernel_size=3,stride=2,padding='SAME')
conv2 = ly.conv2d(conv1,256,kernel_size=5,stride=1,padding='SAME',activation_fn=tf.nn.relu,weights_initializer=tf.random_normal_initializer(0, 0.02))
conv2 = self.bo_batch_norm(conv2, self.is_train)
conv2 = ly.max_pool2d(conv2,kernel_size=3,stride=2,padding='SAME')
conv3 = ly.conv2d(conv2,384,kernel_size=3,stride=1,padding='SAME',activation_fn=tf.nn.relu,weights_initializer=tf.random_normal_initializer(0, 0.02))
conv3 = self.bo_batch_norm(conv3, self.is_train)
#conv3 = ly.max_pool2d(conv3,kernel_size=3,stride=2,padding='SAME')
conv4 = ly.conv2d(conv3,384,kernel_size=3,stride=1,padding='SAME',activation_fn=tf.nn.relu,weights_initializer=tf.random_normal_initializer(0, 0.02))
conv4 = self.bo_batch_norm(conv4, self.is_train)
#conv4 = ly.max_pool2d(conv4,kernel_size=3,stride=2,padding='SAME')
conv5 = ly.conv2d(conv4,256,kernel_size=3,stride=1,padding='SAME',activation_fn=tf.nn.relu,weights_initializer=tf.random_normal_initializer(0, 0.02))
conv5 = self.bo_batch_norm(conv5, self.is_train)
conv5 = ly.max_pool2d(conv5,kernel_size=3,stride=2,padding='SAME')
flat = ly.flatten(conv5)
fc1 = ly.fully_connected(flat, 4096, activation_fn=tf.nn.relu)
fc2 = ly.fully_connected(fc1, 4096, activation_fn=tf.nn.relu)
'''
conv1 = ly.conv2d(
image,
96,
kernel_size=3,
padding='VALID',
stride=2,
activation_fn=tf.nn.relu,
weights_initializer=tf.contrib.layers.xavier_initializer())
conv1 = ly.max_pool2d(conv1,
kernel_size=3,
padding='VALID',
stride=2)
conv2 = ly.conv2d(
conv1,
256,
kernel_size=5,
padding='SAME',
stride=1,
activation_fn=tf.nn.relu,
weights_initializer=tf.contrib.layers.xavier_initializer())
conv2 = ly.max_pool2d(conv2,
kernel_size=3,
padding='SAME',
stride=2)
with tf.variable_scope('subenet_first_group'):
with tf.variable_scope('branch_1'):
pool_1 = ly.max_pool2d(conv2,
kernel_size=3,
padding='SAME',
stride=2)
conv_1 = ly.conv2d(pool_1,
64,
kernel_size=3,
padding='SAME',
stride=2,
weights_initializer=tf.contrib.layers.
xavier_initializer())
with tf.variable_scope('branch_2'):
reduce_3 = ly.conv2d(conv2,
64,
kernel_size=1,
padding='SAME',
stride=2,
weights_initializer=tf.contrib.layers.
xavier_initializer())
conv_3 = ly.conv2d(reduce_3,
64,
kernel_size=3,
padding='SAME',
stride=2,
weights_initializer=tf.contrib.layers.
xavier_initializer())
with tf.variable_scope('branch_3'):
reduce_5 = ly.conv2d(conv2,
64,
kernel_size=1,
padding='SAME',
stride=2,
weights_initializer=tf.contrib.layers.
xavier_initializer())
conv_5 = ly.conv2d(reduce_5,
64,
kernel_size=3,
padding='SAME',
stride=2,
weights_initializer=tf.contrib.layers.
xavier_initializer())
output = tf.concat([conv_1, conv_3, conv_5], axis=2)
pool3_a = ly.max_pool2d(output,
kernel_size=3,
padding='SAME',
stride=2)
flat_first = ly.flatten(pool3_a)
flat_first = ly.fully_connected(
flat_first,
64,
activation_fn=tf.nn.relu,
weights_initializer=tf.contrib.layers.xavier_initializer())
with tf.variable_scope('subnet_second_group'):
with tf.variable_scope('branch_1'):
pool_1 = ly.max_pool2d(conv2,
kernel_size=3,
padding='SAME',
stride=2)
conv_1 = ly.conv2d(pool_1,
64,
kernel_size=3,
padding='SAME',
stride=2,
weights_initializer=tf.contrib.layers.
xavier_initializer())
with tf.variable_scope('branch_2'):
reduce_3 = ly.conv2d(conv2,
64,
kernel_size=1,
padding='SAME',
stride=2,
weights_initializer=tf.contrib.layers.
xavier_initializer())
conv_3 = ly.conv2d(reduce_3,
64,
kernel_size=3,
padding='SAME',
stride=2,
weights_initializer=tf.contrib.layers.
xavier_initializer())
with tf.variable_scope('branch_3'):
reduce_5 = ly.conv2d(conv2,
64,
kernel_size=1,
padding='SAME',
stride=2,
weights_initializer=tf.contrib.layers.
xavier_initializer())
conv_5 = ly.conv2d(reduce_5,
64,
kernel_size=3,
padding='SAME',
stride=2,
weights_initializer=tf.contrib.layers.
xavier_initializer())
output = tf.concat([conv_1, conv_3, conv_5], axis=2)
pool3_a = ly.max_pool2d(output,
kernel_size=3,
padding='SAME',
stride=2)
flat_second = ly.flatten(pool3_a)
flat_second = ly.fully_connected(
flat_second,
64,
activation_fn=tf.nn.relu,
weights_initializer=tf.contrib.layers.xavier_initializer())
return flat_first, flat_second
import tensorflow as tf
import tensorflow.contrib.layers as layers
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("data/mnist", one_hot=True)
num_train = mnist.train.num_examples
num_test = mnist.test.num_examples
batch_size = 100
num_train_batches = num_train / batch_size
num_test_batches = num_test / batch_size
num_epochs = 10
x = tf.placeholder(tf.float32, shape=[None, 784])
y_ = tf.placeholder(tf.float32, shape=[None, 10])
x_img = tf.reshape(x, [-1,28,28,1])
conv1 = layers.max_pool2d(
layers.convolution2d(x_img, 32, [5, 5]),
[2, 2], [2, 2])
conv2 = layers.max_pool2d(
layers.convolution2d(conv1, 64, [5, 5]),
[2, 2], [2, 2])
y = layers.fully_connected(layers.flatten(conv2),
10, activation_fn=tf.nn.softmax)
cross_entropy = tf.reduce_mean(-tf.reduce_sum(y_*tf.log(y), reduction_indices=[1]))
train_step = tf.train.AdamOptimizer().minimize(cross_entropy)
correct = tf.equal(tf.argmax(y,1), tf.argmax(y_,1))
accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))
import time
init = tf.initialize_all_variables()
with tf.Session() as sess:
def deep_g(self, image, reuse=False):
dim = 32
with tf.variable_scope('compressor') as scope:
if reuse:
scope.reuse_variables()
conv1 = ly.conv2d(
image,
64,
kernel_size=3,
stride=1,
padding='SAME',
activation_fn=tf.nn.relu,
weights_initializer=tf.contrib.layers.xavier_initializer(),
normalizer_fn=ly.batch_norm)
conv2 = ly.conv2d(
conv1,
64,
kernel_size=3,
stride=1,
padding='SAME',
activation_fn=tf.nn.relu,
weights_initializer=tf.contrib.layers.xavier_initializer(),
normalizer_fn=ly.batch_norm)
pool2 = ly.max_pool2d(conv2, 2, padding='SAME')
dropout2 = tf.nn.dropout(pool2, keep_prob=self.keep_prob)
conv3 = ly.conv2d(
dropout2,
128,
kernel_size=3,
stride=1,
padding='SAME',
activation_fn=tf.nn.relu,
weights_initializer=tf.contrib.layers.xavier_initializer(),
normalizer_fn=ly.batch_norm)
conv4 = ly.conv2d(
conv3,
128,
kernel_size=3,
stride=1,
padding='SAME',
activation_fn=tf.nn.relu,
weights_initializer=tf.contrib.layers.xavier_initializer(),
normalizer_fn=ly.batch_norm)
pool4 = ly.max_pool2d(conv4, 2, padding='SAME')
dropout4 = tf.nn.dropout(pool4, keep_prob=self.keep_prob)
conv5 = ly.conv2d(
dropout4,
256,
kernel_size=3,
stride=1,
padding='SAME',
activation_fn=tf.nn.relu,
weights_initializer=tf.contrib.layers.xavier_initializer(),
normalizer_fn=ly.batch_norm)
conv6 = ly.conv2d(
conv5,
256,
kernel_size=3,
stride=1,
padding='SAME',
activation_fn=tf.nn.relu,
weights_initializer=tf.contrib.layers.xavier_initializer(),
normalizer_fn=ly.batch_norm)
conv7 = ly.conv2d(
conv6,
256,
kernel_size=3,
stride=1,
padding='SAME',
activation_fn=tf.nn.relu,
weights_initializer=tf.contrib.layers.xavier_initializer(),
normalizer_fn=ly.batch_norm)
pool7 = ly.max_pool2d(conv7, 2, padding='SAME')
dropout7 = tf.nn.dropout(pool7, keep_prob=self.keep_prob)
conv8 = ly.conv2d(
dropout7,
512,
kernel_size=3,
stride=1,
padding='SAME',
activation_fn=tf.nn.relu,
weights_initializer=tf.contrib.layers.xavier_initializer(),
normalizer_fn=ly.batch_norm)
conv9 = ly.conv2d(
conv8,
512,
kernel_size=3,
stride=1,
padding='SAME',
activation_fn=tf.nn.relu,
weights_initializer=tf.contrib.layers.xavier_initializer(),
normalizer_fn=ly.batch_norm)
conv10 = ly.conv2d(
conv9,
512,
kernel_size=3,
stride=1,
padding='SAME',
activation_fn=tf.nn.relu,
weights_initializer=tf.contrib.layers.xavier_initializer(),
normalizer_fn=ly.batch_norm)
pool10 = ly.max_pool2d(conv10, 2, padding='SAME')
dropout10 = tf.nn.dropout(pool10, keep_prob=self.keep_prob)
conv11 = ly.conv2d(
dropout10,
512,
kernel_size=3,
stride=1,
padding='SAME',
activation_fn=tf.nn.relu,
weights_initializer=tf.contrib.layers.xavier_initializer(),
normalizer_fn=ly.batch_norm)
conv12 = ly.conv2d(
conv11,
512,
kernel_size=3,
stride=1,
padding='SAME',
activation_fn=tf.nn.relu,
weights_initializer=tf.contrib.layers.xavier_initializer(),
normalizer_fn=ly.batch_norm)
conv13 = ly.conv2d(
conv12,
512,
kernel_size=3,
stride=1,
padding='SAME',
activation_fn=tf.nn.relu,
weights_initializer=tf.contrib.layers.xavier_initializer(),
normalizer_fn=ly.batch_norm)
pool13 = ly.max_pool2d(conv13, 2, padding='SAME')
dropout13 = tf.nn.dropout(pool13, keep_prob=self.keep_prob)
flat = ly.flatten(dropout13)
flat = ly.fully_connected(
flat,
4096,
activation_fn=tf.nn.relu,
weights_initializer=tf.random_normal_initializer(0, 0.02))
return flat
def create_model(self, input):
with tf.variable_scope('conv1'):
conv1 = conv2d(input,
num_outputs=64,
kernel_size=[7, 7],
stride=2,
padding='SAME',
activation_fn=None)
conv1 = tf.layers.batch_normalization(conv1)
conv1 = tf.nn.relu(conv1)
self.conv1 = conv1
with tf.variable_scope('conv2'):
conv2 = max_pool2d(conv1,
kernel_size=[3, 3],
stride=2,
padding='SAME')
if self.model_type is "18" or self.model_type is "34":
conv2 = self.repeat_residual_blocks(
repeat=2,
x=conv2,
block=self.residual_block_a,
num_outputs=[64, 64],
kernel_sizes=[[3, 3], [3, 3]],
pool=False)
if self.model_type is "34":
conv2 = self.repeat_residual_blocks(
repeat=2,
x=conv2,
block=self.residual_block_a,
num_outputs=[64],
kernel_sizes=[[3, 3][3, 3]],
pool=False)
elif self.model_type is "50" or self.model_type is "101" or self.model_type is "152" or self.model_type is "200":
conv2 = self.repeat_residual_blocks(
repeat=3,
x=conv2,
block=self.residual_block_b,
num_outputs=[64, 64, 256],
kernel_sizes=[[1, 1], [3, 3], [1, 1]],
pool=False)
self.conv2 = conv2
with tf.variable_scope('conv3'):
if self.model_type is "18" or self.model_type is "34":
conv3 = self.repeat_residual_blocks(
repeat=2,
x=conv2,
block=self.residual_block_a,
num_outputs=[128, 128],
kernel_sizes=[[3, 3], [3, 3]],
pool=True)
if self.model_type is "34":
conv3 = self.repeat_residual_blocks(
repeat=2,
x=conv3,
block=self.residual_block_a,
num_outputs=[128, 128],
kernel_sizes=[[3, 3], [3, 3]],
pool=False)
elif self.model_type is "50" or self.model_type is "101" or self.model_type is "152" or self.model_type is "200":
conv3 = self.repeat_residual_blocks(
repeat=4,
x=conv2,
block=self.residual_block_b,
num_outputs=[128, 128, 512],
kernel_sizes=[[1, 1], [3, 3], [1, 1]],
pool=True)
if self.model_type is "152":
conv3 = self.repeat_residual_blocks(
repeat=4,
x=conv3,
block=self.residual_block_b,
num_outputs=[128, 128, 512],
kernel_sizes=[[1, 1], [3, 3], [1, 1]],
pool=False)
if self.model_type is "200":
conv3 = self.repeat_residual_blocks(
repeat=16,
x=conv3,
block=self.residual_block_b,
num_outputs=[128, 128, 512],
kernel_sizes=[[1, 1], [3, 3], [1, 1]],
pool=False)
self.conv3 = conv3
with tf.variable_scope('conv4'):
if self.model_type is "18" or self.model_type is "34":
conv4 = self.repeat_residual_blocks(
repeat=2,
x=conv3,
block=self.residual_block_a,
num_outputs=[256, 256],
kernel_sizes=[[3, 3], [3, 3]],
pool=True)
if self.model_type is "34":
conv4 = self.repeat_residual_blocks(
repeat=4,
x=conv4,
block=self.residual_block_a,
num_outputs=[256, 256],
kernel_sizes=[[3, 3], [3, 3]],
pool=False)
elif self.model_type is "50" or self.model_type is "101" or self.model_type is "152" or self.model_type is "200":
conv4 = self.repeat_residual_blocks(
repeat=6,
x=conv3,
block=self.residual_block_b,
num_outputs=[256, 256, 1024],
kernel_sizes=[[1, 1], [3, 3], [1, 1]],
pool=True)
if self.model_type is "101" or self.model_type is "152" or self.model_type is "200":
conv4 = self.repeat_residual_blocks(
repeat=17,
x=conv4,
block=self.residual_block_b,
num_outputs=[256, 256, 1024],
kernel_sizes=[[1, 1], [3, 3], [1, 1]],
pool=False)
if self.model_type is "152" or self.model_type is "200":
conv4 = self.repeat_residual_blocks(
repeat=77,
x=conv4,
block=self.residual_block_b,
num_outputs=[256, 256, 1024],
kernel_sizes=[[1, 1], [3, 3], [1, 1]],
pool=False)
self.conv4 = conv4
with tf.variable_scope('conv5'):
if self.model_type is "18" or self.model_type is "34":
conv5 = self.repeat_residual_blocks(
repeat=2,
x=conv4,
block=self.residual_block_a,
num_outputs=[512, 512],
kernel_sizes=[[3, 3], [3, 3]],
pool=True)
if self.model_type is "34":
conv5 = self.repeat_residual_blocks(
repeat=1,
x=conv5,
block=self.residual_block_a,
num_outputs=[512, 512],
kernel_sizes=[[3, 3], [3, 3]],
pool=True)
elif self.model_type is "50" or self.model_type is "101" or self.model_type is "152" or self.model_type is "200":
conv5 = self.repeat_residual_blocks(
repeat=3,
x=conv4,
block=self.residual_block_b,
num_outputs=[512, 512, 2048],
kernel_sizes=[[1, 1], [3, 3], [1, 1]],
pool=True)
self.conv5 = conv5
with tf.variable_scope('before_final'):
avg_pool = avg_pool2d(conv5,
kernel_size=[3, 3],
stride=2,
padding='SAME')
flat = flatten(avg_pool)
self.flat = flat
with tf.variable_scope('final'):
self.final_out = fully_connected(flat,
num_outputs=self.num_classes,
activation_fn=None)
return [self.final_out]
def build(inputs, labels, weights, is_training=True):
vgg_layers, vgg_layer_names = read_vgg_init(FLAGS.vgg_init_dir)
weight_decay = 5e-4
bn_params = {
# Decay for the moving averages.
'decay': 0.999,
'center': True,
'scale': True,
# epsilon to prevent 0s in variance.
'epsilon': 0.001,
# None to force the updates
'updates_collections': None,
'is_training': is_training,
}
with tf.contrib.framework.arg_scope(
[layers.convolution2d],
kernel_size=3,
stride=1,
padding='SAME',
rate=1,
activation_fn=tf.nn.relu,
# normalizer_fn=layers.batch_norm, normalizer_params=bn_params,
# weights_initializer=layers.variance_scaling_initializer(),
normalizer_fn=None,
weights_initializer=None,
weights_regularizer=layers.l2_regularizer(weight_decay)):
net = layers.convolution2d(inputs, 64, scope='conv1_1')
net = layers.convolution2d(net, 64, scope='conv1_2')
net = layers.max_pool2d(net, 2, 2, scope='pool1')
net = layers.convolution2d(net, 128, scope='conv2_1')
net = layers.convolution2d(net, 128, scope='conv2_2')
net = layers.max_pool2d(net, 2, 2, scope='pool2')
net = layers.convolution2d(net, 256, scope='conv3_1')
net = layers.convolution2d(net, 256, scope='conv3_2')
net = layers.convolution2d(net, 256, scope='conv3_3')
net = layers.max_pool2d(net, 2, 1, scope='pool3', padding='SAME')
paddings = [[0, 0], [0, 0]]
crops = [[0, 0], [0, 0]]
block_size = 2
net = tf.space_to_batch(net, paddings=paddings, block_size=block_size)
net = layers.convolution2d(net, 512, scope='conv4_1')
net = layers.convolution2d(net, 512, scope='conv4_2')
net = layers.convolution2d(net, 512, scope='conv4_3')
net = tf.batch_to_space(net, crops=crops, block_size=block_size)
net = layers.max_pool2d(net, 2, 1, scope='pool4', padding='SAME')
block_size = 4
net = tf.space_to_batch(net, paddings=paddings, block_size=block_size)
net = layers.convolution2d(net, 512, scope='conv5_1')
net = layers.convolution2d(net, 512, scope='conv5_2')
net = layers.convolution2d(net, 512, scope='conv5_3')
net = tf.batch_to_space(net, crops=crops, block_size=block_size)
with tf.contrib.framework.arg_scope(
[layers.convolution2d],
stride=1,
padding='SAME',
weights_initializer=layers.variance_scaling_initializer(),
activation_fn=tf.nn.relu,
normalizer_fn=layers.batch_norm,
normalizer_params=bn_params,
weights_regularizer=layers.l2_regularizer(FLAGS.weight_decay)):
net = layers.convolution2d(net,
512,
kernel_size=3,
scope='conv6_1',
rate=8)
logits = layers.convolution2d(net,
FLAGS.num_classes,
1,
padding='SAME',
activation_fn=None,
scope='unary_2',
rate=2)
print('logits', logits.get_shape())
logits = tf.image.resize_bilinear(logits,
[FLAGS.img_height, FLAGS.img_width],
name='resize_score')
loss = get_loss(logits, labels, weights, is_training=is_training)
if is_training:
init_op, init_feed = create_init_op(vgg_layers)
return logits, loss, init_op, init_feed
return logits, loss
def get_conv_model(features, labels, mode, params):
parent_scope = "cnn" # TODO Need to have two: one for expand, one for conquer
# features = _get_feature_dict(features)
head = params.get("head")
feature_columns = params.get("feature_columns")
activation_fn = params.get("activation_fn")
dropout = params.get("dropout")
learning_rate = params.get("learning_rate")
optimizer = params.get("optimizer")
# with variable_scope.variable_scope(
# parent_scope + "/input_from_feature_columns",
# values=features.values()) as scope:
# net = layers.input_from_feature_columns(
# columns_to_tensors=features,
# feature_columns=feature_columns,
# weight_collections=[parent_scope],
# scope=scope)
with variable_scope.variable_scope(
parent_scope + "/convlayer_1",
values=[features]) as scope:
net = layers.conv2d(features,
num_outputs=32,
kernel_size=3,
variables_collections=[parent_scope],
scope=scope)
net = layers.max_pool2d(net, 2,
stride=1,
padding='SAME')
with variable_scope.variable_scope(
parent_scope + "/convlayer_2",
values=[features]) as scope:
net = layers.conv2d(features,
num_outputs=64,
kernel_size=5,
padding='VALID',
variables_collections=[parent_scope],
scope=scope)
# net = layers.max_pool2d(net, 1,
# stride=1,
# padding='SAME')
#
# with variable_scope.variable_scope(
# parent_scope + "/max_pool_1",
# values=[net]) as scope:
shape = net.get_shape()
net = tf.reshape(net, [-1, shape[3].value], name="reshape_1")
hidden_units = [256, 128]
for layer_id, num_hidden_units in enumerate(hidden_units):
with variable_scope.variable_scope(
parent_scope + "/hiddenlayer_%d" % layer_id,
values=[net]) as scope:
net = layers.fully_connected(
net,
num_hidden_units,
activation_fn=activation_fn,
variables_collections=[parent_scope],
scope=scope)
if dropout is not None and mode == model_fn.ModeKeys.TRAIN:
net = layers.dropout(
net,
keep_prob=(1.0 - dropout))
with variable_scope.variable_scope(
parent_scope + "/logits",
values=[net]) as scope:
logits = layers.fully_connected(
net,
head.logits_dimension,
activation_fn=None,
variables_collections=[parent_scope],
scope=scope)
def _train_op_fn(loss):
"""Returns the op to optimize the loss."""
return optimizers.optimize_loss(
loss=loss,
global_step=contrib_variables.get_global_step(),
learning_rate=learning_rate,
optimizer=optimizer,
name=parent_scope,
# Empty summaries to prevent optimizers from logging the training_loss.
summaries=[])
return head.head_ops(features, labels, mode, _train_op_fn, logits)
def Alex_net(self, input, name_1, name_2):
with tf.variable_scope(name_1):
conv1 = ly.conv2d(
input,
96,
kernel_size=11,
stride=4,
padding='SAME',
activation_fn=tf.nn.relu,
weights_initializer=tf.contrib.layers.xavier_initializer())
#conv1 = self.bo_batch_norm(conv1, self.is_train)
conv1 = ly.max_pool2d(conv1,
kernel_size=3,
stride=2,
padding='SAME')
conv2 = ly.conv2d(
conv1,
256,
kernel_size=5,
stride=1,
padding='SAME',
activation_fn=tf.nn.relu,
weights_initializer=tf.contrib.layers.xavier_initializer())
#conv2 = self.bo_batch_norm(conv2, self.is_train)
conv2 = ly.max_pool2d(conv2,
kernel_size=3,
stride=2,
padding='SAME')
conv3 = ly.conv2d(
conv2,
384,
kernel_size=3,
stride=1,
padding='SAME',
activation_fn=tf.nn.relu,
weights_initializer=tf.contrib.layers.xavier_initializer())
#conv3 = self.bo_batch_norm(conv3, self.is_train)
#conv3 = ly.max_pool2d(conv3,kernel_size=3,stride=2,padding='SAME')
conv4 = ly.conv2d(
conv3,
384,
kernel_size=3,
stride=1,
padding='SAME',
activation_fn=tf.nn.relu,
weights_initializer=tf.contrib.layers.xavier_initializer())
#conv4 = self.bo_batch_norm(conv4, self.is_train)
#conv4 = ly.max_pool2d(conv4,kernel_size=3,stride=2,padding='SAME')
conv5 = ly.conv2d(
conv4,
256,
kernel_size=3,
stride=1,
padding='SAME',
activation_fn=tf.nn.relu,
weights_initializer=tf.contrib.layers.xavier_initializer())
#conv5 = self.bo_batch_norm(conv5, self.is_train)
conv5 = ly.max_pool2d(conv5,
kernel_size=3,
stride=2,
padding='SAME')
flat = ly.flatten(conv5)
print(conv5)
with tf.variable_scope(name_2):
fc1 = ly.fully_connected(
flat,
4096,
activation_fn=tf.nn.relu,
weights_initializer=tf.contrib.layers.xavier_initializer())
fc2 = ly.fully_connected(
fc1,
4096,
activation_fn=tf.nn.relu,
weights_initializer=tf.contrib.layers.xavier_initializer())
logit = ly.fully_connected(
fc2,
2,
activation_fn=None,
weights_initializer=tf.contrib.layers.xavier_initializer())
return flat, logit