def comparator_caffenet(input_image, reuse=False, trainable=False):
"""
Compare the feature in relu5 feature map
"""
with tf.variable_scope('comparator', reuse=reuse) as vs:
# input_image = tf.placeholder(tf.float32, shape=(None, 227, 227, 3), name='input_image')
assert input_image.get_shape().as_list()[1:] == [227, 227, 3]
relu1 = relu(
conv(input_image,
11,
11,
96,
4,
4,
pad='VALID',
name='conv1',
trainable=trainable))
pool1 = max_pool(relu1, 3, 3, 2, 2, pad='VALID', name='pool1')
norm1 = lrn(pool1, 2, 2e-05, 0.75, name='norm1')
relu2 = relu(
conv(norm1,
5,
5,
256,
1,
1,
group=2,
name='conv2',
trainable=trainable))
pool2 = max_pool(relu2, 3, 3, 2, 2, pad='VALID', name='pool2')
norm2 = lrn(pool2, 2, 2e-05, 0.75, name='norm2')
relu3 = relu(
conv(norm2, 3, 3, 384, 1, 1, name='conv3', trainable=trainable))
relu4 = relu(
conv(relu3,
3,
3,
384,
1,
1,
group=2,
name='conv4',
trainable=trainable))
relu5 = relu(
conv(relu4,
3,
3,
256,
1,
1,
group=2,
name='conv5',
trainable=trainable))
variables = tf.contrib.framework.get_variables(vs)
return relu5, variables
def discriminator_image(input_image,
real_input_label,
fake_input_label,
num_train_classes,
name='image',
reuse=False):
with tf.variable_scope('discriminator_%s' % (name), reuse=reuse) as vs:
dconv1 = relu(
conv(input_image, 7, 7, 32, 4, 4, pad='VALID', name='dconv1'))
dconv2 = relu(conv(dconv1, 5, 5, 64, 1, 1, pad='VALID', name='dconv2'))
dconv3 = relu(conv(dconv2, 3, 3, 128, 2, 2, pad='VALID',
name='dconv3'))
dconv4 = relu(conv(dconv3, 3, 3, 256, 1, 1, pad='VALID',
name='dconv4'))
dconv5 = relu(conv(dconv4, 3, 3, 256, 2, 2, pad='VALID',
name='dconv5'))
dpool5 = avg_pool(dconv5, 11, 11, 11, 11, name='dpool5')
dpool5_reshape = tf.reshape(dpool5, [-1, 256], name='dpool5_reshape')
# label information
real_label_feat = tf.one_hot(real_input_label, depth=num_train_classes)
fake_label_feat = tf.one_hot(fake_input_label, depth=num_train_classes)
label_feat = tf.concat([real_label_feat, fake_label_feat], axis=0)
# SAVE GPU MEMORY
Ffc1 = relu(fc(label_feat, 128, name='Ffc1'))
Ffc2 = relu(fc(Ffc1, 128, name='Ffc2'))
concat5 = tf.concat([dpool5_reshape, Ffc2], axis=1, name='concat5')
drop5 = tf.nn.dropout(concat5, keep_prob=0.5, name='drop5')
# SAVE GPU MEMORY
dfc6 = relu(fc(drop5, 256, name='dfc6'))
dfc7 = fc(dfc6, 1, name='dfc7')
variables = tf.contrib.framework.get_variables(vs)
return dfc7, variables
def discriminator_patch(image,
map_feature,
df_dim,
reuse=False,
dropout=False,
keep_prob=1.0,
name="default"):
assert len(map_feature.get_shape().as_list()) == 3
map_feature = tf.nn.l2_normalize(map_feature,
dim=2,
name='normed_map_feature')
with tf.variable_scope("discriminator_patch_%s" % (name),
reuse=reuse) as vs:
h0 = leaky_relu(instance_norm(
conv(image,
4,
4,
df_dim,
2,
2,
name='d_h0_conv',
init='random',
biased=False)),
alpha=0.2)
h0 = tf.concat([
tf.nn.l2_normalize(h0, dim=3, name='l2_normalized_h0'),
tf.tile(map_feature[:, :, None, :], [1, 128, 128, 1])
],
axis=3)
# h0 is (128 x 128 x self.df_dim)
h1 = leaky_relu(instance_norm(
conv(h0,
4,
4,
df_dim * 2,
2,
2,
name='d_h1_conv',
init='random',
biased=False), 'd_bn1'),
alpha=0.2)
# h1 is (64 x 64 x self.df_dim*2)
h2 = leaky_relu(instance_norm(
conv(h1,
4,
4,
df_dim * 4,
2,
2,
name='d_h2_conv',
init='random',
biased=False), 'd_bn2'),
alpha=0.2)
# h2 is (32x 32 x self.df_dim*4)
h3 = leaky_relu(instance_norm(
conv(h2,
4,
4,
df_dim * 8,
1,
1,
name='d_h3_conv',
init='random',
biased=False), 'd_bn3'),
alpha=0.2)
# h3 is (32 x 32 x self.df_dim*8)
h4 = relu(conv(h3, 4, 4, 1, 1, 1, name='d_h3_pred'))
# h4 is (32 x 32 x 1)
variables = tf.contrib.framework.get_variables(vs)
return h4, variables
def encoder_caffenet(input_image, feat='fc6', reuse=False, trainable=False):
with tf.variable_scope('encoder_caffenet', reuse=reuse) as vs:
assert input_image.get_shape().as_list()[1:] == [227, 227, 3]
# input_image = tf.placeholder(tf.float32, shape=(None, 227, 227, 3), name='input_image')
relu1 = relu(
conv(input_image,
11,
11,
96,
4,
4,
pad='VALID',
name='conv1',
trainable=trainable))
pool1 = max_pool(relu1, 3, 3, 2, 2, pad='VALID', name='pool1')
norm1 = lrn(pool1, 2, 2e-05, 0.75, name='norm1')
relu2 = relu(
conv(norm1,
5,
5,
256,
1,
1,
group=2,
name='conv2',
trainable=trainable))
pool2 = max_pool(relu2, 3, 3, 2, 2, pad='VALID', name='pool2')
norm2 = lrn(pool2, 2, 2e-05, 0.75, name='norm2')
relu3 = relu(
conv(norm2, 3, 3, 384, 1, 1, name='conv3', trainable=trainable))
relu4 = relu(
conv(relu3,
3,
3,
384,
1,
1,
group=2,
name='conv4',
trainable=trainable))
relu5 = relu(
conv(relu4,
3,
3,
256,
1,
1,
group=2,
name='conv5',
trainable=trainable))
pool5 = max_pool(relu5, 3, 3, 2, 2, pad='VALID', name='pool5')
fc6 = relu(fc(pool5, 4096, name='fc6', trainable=trainable))
fc7 = relu(fc(fc6, 4096, name='fc7', trainable=trainable))
variables = tf.contrib.framework.get_variables(vs)
if feat == 'fc6':
return fc6, variables
elif feat == 'fc7':
return fc7, variables
else:
raise NotImplementedError
def __init__(self, imageWidth, imageHeight, activation='relu', batchnorm=False, \
dropout=0.0, flownet_weights_path=None, numLSTMCells=1, hidden_units_imu=None, hidden_units_LSTM=None, \
numFC=2, FC_dims=None):
super(VINet, self).__init__()
# Check if input image width and height are feasible
self.imageWidth = int(imageWidth)
self.imageHeight = int(imageHeight)
if self.imageWidth < 64 or self.imageHeight < 64:
raise ValueError(
'The width and height for an input image must be at least 64 px.'
)
# Compute the size of the LSTM input feature vector.
# There are 6 conv stages (some stages have >1 conv layers), which effectively reduce an
# image to 1/64 th of its initial dimensions. Further, the final conv layer has 1024
# filters, hence, numConcatFeatures = 1024 * (wd/64) * (ht/64) = (wd * ht) / 4
self.numConcatFeatures = int((self.imageWidth * self.imageHeight) / 4)
# Activation functions to be used in the network
self.activation = activation
# Whether or not batchnorm is required
self.batchNorm = batchnorm
# Whether or not dropout is required
if dropout <= 0.0:
self.dropout = False
else:
# Specify the drop_ratio
self.dropout = True
self.drop_ratio = dropout
self.numLSTMCells = numLSTMCells
self.hidden_units_LSTM = hidden_units_LSTM
self.hidden_units_imu = hidden_units_imu
# Path to FlowNet weights
if flownet_weights_path is not None:
self.use_flownet = True
self.flownet_weights_path = flownet_weights_path
else:
self.use_flownet = False
"""False
Initialize variables required for the network
"""
# If we're using batchnorm, do not use bias for the conv layers
self.bias = not self.batchNorm
self.conv1 = conv(self.batchNorm, 1, 64, kernel_size=7, stride=2)
self.conv2 = conv(self.batchNorm, 64, 128, kernel_size=5, stride=2)
self.conv3 = conv(self.batchNorm, 128, 256, kernel_size=5, stride=2)
self.conv_redir = conv(self.batchNorm,
256,
32,
kernel_size=1,
stride=1)
self.conv3_1 = conv(self.batchNorm, 473, 256)
self.conv4 = conv(self.batchNorm, 256, 512, stride=2)
self.conv4_1 = conv(self.batchNorm, 512, 512)
self.conv5 = conv(self.batchNorm, 512, 512, stride=2)
self.conv5_1 = conv(self.batchNorm, 512, 512)
self.conv6 = conv(self.batchNorm, 512, 1024, stride=2)
self.conv6_1 = conv(self.batchNorm, 1024, 1024)
self.rnnIMU = nn.LSTM(
input_size=6,
hidden_size=6, #hidden_units_imu[0],
num_layers=2,
batch_first=True)
self.rnnIMU.cuda()
self.rnn = nn.LSTM(input_size=98317,
hidden_size=hidden_units_LSTM[0],
num_layers=2,
batch_first=True)
self.rnn.cuda()
self.fc1 = nn.Linear(self.hidden_units_LSTM[self.numLSTMCells - 1],
128)
self.fc1.cuda()
# self.fc1 = nn.Linear(1024, 128)
self.fc2 = nn.Linear(128, 32)
self.fc2.cuda()
self.fc_out = nn.Linear(32, 6)
self.fc_out.cuda()
def classifier_caffenet(input_image,
output,
reuse=False,
trainable=False,
dropout=False,
keep_prob=1.0):
with tf.variable_scope('classifier', reuse=reuse) as vs:
# input_image = tf.placeholder(tf.float32, shape=(None, 227, 227, 3), name='input_image')
# assert input_image.get_shape().as_list()[1:] == [227, 227, 3]
relu1 = relu(
conv(input_image,
11,
11,
96,
4,
4,
pad='VALID',
name='conv1',
trainable=trainable))
pool1 = max_pool(relu1, 3, 3, 2, 2, pad='VALID', name='pool1')
norm1 = lrn(pool1, 2, 2e-05, 0.75, name='norm1')
relu2 = relu(
conv(norm1,
5,
5,
256,
1,
1,
group=2,
name='conv2',
trainable=trainable))
pool2 = max_pool(relu2, 3, 3, 2, 2, pad='VALID', name='pool2')
norm2 = lrn(pool2, 2, 2e-05, 0.75, name='norm2')
relu3 = relu(
conv(norm2, 3, 3, 384, 1, 1, name='conv3', trainable=trainable))
relu4 = relu(
conv(relu3,
3,
3,
384,
1,
1,
group=2,
name='conv4',
trainable=trainable))
relu5 = relu(
conv(relu4,
3,
3,
256,
1,
1,
group=2,
name='conv5',
trainable=trainable))
pool5 = max_pool(relu5, 3, 3, 2, 2, pad='VALID', name='pool5')
fc6 = relu(fc(pool5, 4096, name='fc6', trainable=trainable))
if dropout:
fc6 = tf.nn.dropout(fc6, keep_prob=keep_prob)
fc7 = relu(fc(fc6, 4096, name='fc7', trainable=trainable))
if dropout:
fc7 = tf.nn.dropout(fc7, keep_prob=keep_prob)
fc8 = fc(fc7, output, name='output', trainable=trainable)
variables = tf.contrib.framework.get_variables(vs)
return fc8, variables
def generator_unet(image,
map_feature,
gf_dim,
reuse=False,
dropout=False,
keep_prob=1.0,
name="default"):
assert len(map_feature.get_shape().as_list()) == 3
map_feature = tf.nn.l2_normalize(map_feature,
dim=2,
name='normed_map_feature')
with tf.variable_scope("generator_unet_%s" % (name), reuse=reuse) as vs:
# image is 256 x 256 x input_c_dim
# if reuse:
# tf.get_variable_scope().reuse_variables()
# else:
# assert tf.get_variable_scope().reuse is False
assert image.get_shape().as_list()[1:] == [256, 256, 3]
e1 = instance_norm(
conv(image,
4,
4,
gf_dim,
2,
2,
name='g_e1_conv',
init='random',
biased=False), 'g_bn_e1')
e1 = tf.concat([
tf.nn.l2_normalize(e1, dim=3, name='l2_normalized_e1'),
tf.tile(map_feature[:, :, None, :], [1, 128, 128, 1])
],
axis=3)
# test_e1 = instance_norm(e1)
# e1 is (128 x 128 x self.gf_dim)
e2 = instance_norm(
conv(leaky_relu(e1, alpha=0.2),
4,
4,
gf_dim * 2,
2,
2,
name='g_e2_conv',
init='random',
biased=False), 'g_bn_e2')
# e2 is (64 x 64 x self.gf_dim*2)
# e2 = tf.concat([tf.nn.l2_normalize(e2, dim=3, name='l2_normalized_e2'), tf.tile(map_feature[:, :, None, :], [1, 64, 64, 1])], axis=3)
e3 = instance_norm(
conv(leaky_relu(e2, alpha=0.2),
4,
4,
gf_dim * 4,
2,
2,
name='g_e3_conv',
init='random',
biased=False), 'g_bn_e3')
# e3 is (32 x 32 x self.gf_dim*4)
e4 = instance_norm(
conv(leaky_relu(e3, alpha=0.2),
4,
4,
gf_dim * 8,
2,
2,
name='g_e4_conv',
init='random',
biased=False), 'g_bn_e4')
# e4 is (16 x 16 x self.gf_dim*8)
e5 = instance_norm(
conv(leaky_relu(e4, alpha=0.2),
4,
4,
gf_dim * 8,
2,
2,
name='g_e5_conv',
init='random',
biased=False), 'g_bn_e5')
# e5 is (8 x 8 x self.gf_dim*8)
e6 = instance_norm(
conv(leaky_relu(e5, alpha=0.2),
4,
4,
gf_dim * 8,
2,
2,
name='g_e6_conv',
init='random',
biased=False), 'g_bn_e6')
# e6 is (4 x 4 x self.gf_dim*8)
e7 = instance_norm(
conv(leaky_relu(e6, alpha=0.2),
4,
4,
gf_dim * 8,
2,
2,
name='g_e7_conv',
init='random',
biased=False), 'g_bn_e7')
# e7 is (2 x 2 x self.gf_dim*8)
# e8 = instance_norm(conv(leaky_relu(e7, alpha=0.2), 4, 4, gf_dim*8, 2, 2, name='g_e8_conv', init='random', activation=None, biased=False), 'g_bn_e8')
# remove instance norm
e8 = conv(leaky_relu(e7, alpha=0.2),
4,
4,
gf_dim * 8,
2,
2,
name='g_e8_conv',
init='random',
biased=False)
# e8 is (1 x 1 x self.gf_dim*8)
# self.upper_norm_image_feat = tf.nn.l2_normalize(self.upper_image_feat, dim=1, name='upper_norm_image_feat')
# e8 = tf.nn.l2_normalize(e8, dim=3, name='l2_normalized_e8')
# d1 = upconv(tf.nn.relu(tf.concat([e8, map_feature[:, :, None, :]], axis=3)), gf_dim*8, 4, 2, name='g_d1', init='random', activation=None)
d1 = upconv(relu(e8), gf_dim * 8, 4, 2, name='g_d1')
if dropout:
d1 = tf.nn.dropout(d1, keep_prob)
d1 = tf.concat([instance_norm(d1, 'g_bn_d1'), e7], 3)
# d1 is (2 x 2 x self.gf _dim*8*2)
d2 = upconv(relu(d1), gf_dim * 8, 4, 2, name='g_d2', init='random')
if dropout:
d2 = tf.nn.dropout(d2, keep_prob)
d2 = tf.concat([instance_norm(d2, 'g_bn_d2'), e6], 3)
# d2 is (4 x 4 x self.gf_dim*8*2)
d3 = upconv(relu(d2), gf_dim * 8, 4, 2, name='g_d3', init='random')
if dropout:
d3 = tf.nn.dropout(d3, keep_prob)
d3 = tf.concat([instance_norm(d3, 'g_bn_d3'), e5], 3)
# d3 is (8 x 8 x self.gf_dim*8*2)
d4 = upconv(relu(d3), gf_dim * 8, 4, 2, name='g_d4', init='random')
d4 = tf.concat([instance_norm(d4, 'g_bn_d4'), e4], 3)
# d4 is (16 x 16 x self.gf_dim*8*2)
d5 = upconv(relu(d4), gf_dim * 4, 4, 2, name='g_d5', init='random')
d5 = tf.concat([instance_norm(d5, 'g_bn_d5'), e3], 3)
d6 = upconv(d5, gf_dim * 2, 4, 2, name='g_d6', init='random')
d6 = tf.concat([instance_norm(d6, 'g_bn_d6'), e2], 3)
# d6 is (64 x 64 x self.gf_dim*2*2)
d7 = upconv(relu(d6), gf_dim, 4, 2, name='g_d7', init='random')
d7 = tf.concat([instance_norm(d7, 'g_bn_d7'), e1], 3) # one to multi
# d7 is (128 x 128 x self.gf_dim*1*2)
d8 = upconv(relu(d7), 3, 4, 2, name='g_d8', init='random')
# d8 is (256 x 256 x output_c_dim)
variables = tf.contrib.framework.get_variables(vs)
return tf.nn.tanh(d8), variables