def encoder(self, input_image, scope_name="encoder", reuse=False):
with tf.variable_scope(scope_name):
c1 = op.conv2d(input_image,
64,
kernel_h=5,
kernel_w=5,
k_stride=1,
scope_name='conv1')
c1 = op.batch_norm(c1, scope_name='bn1')
self.c1.append(c1)
c2 = op.conv2d(c1,
128,
kernel_h=5,
kernel_w=5,
k_stride=2,
scope_name='conv2')
c2 = op.batch_norm(c2, scope_name='bn2')
self.c2.append(c2)
c3 = op.conv2d(c2,
256,
kernel_h=5,
kernel_w=5,
k_stride=1,
scope_name='conv3')
c3 = op.batch_norm(c3, scope_name='bn3')
self.c3.append(c3)
c4 = op.conv2d(c3,
512,
kernel_h=5,
kernel_w=5,
k_stride=2,
scope_name='conv4')
self.c3.append(c4)
print(c1.shape, c2.shape, c3.shape, c4.shape)
return c4
def build(self, images, train):
with tf.variable_scope(self.scope_name, reuse=tf.AUTO_REUSE):
layers = [images]
with tf.variable_scope("layer0"):
layers.append(conv2d(
layers[-1],
32,
d_h=self.stride,
d_w=self.stride,
k_h=self.kernel,
k_w=self.kernel,
sn_op=None))
layers.append(lrelu(layers[-1]))
with tf.variable_scope("layer1"):
layers.append(conv2d(
layers[-1],
32,
d_h=self.stride,
d_w=self.stride,
k_h=self.kernel,
k_w=self.kernel,
sn_op=None))
layers.append(lrelu(layers[-1]))
layers.append(batch_norm()(layers[-1], train=train))
with tf.variable_scope("layer2"):
layers.append(conv2d(
layers[-1],
64,
d_h=self.stride,
d_w=self.stride,
k_h=self.kernel,
k_w=self.kernel,
sn_op=None))
layers.append(lrelu(layers[-1]))
layers.append(batch_norm()(layers[-1], train=train))
with tf.variable_scope("layer3"):
layers.append(conv2d(
layers[-1],
64,
d_h=self.stride,
d_w=self.stride,
k_h=self.kernel,
k_w=self.kernel,
sn_op=None))
layers.append(lrelu(layers[-1]))
layers.append(batch_norm()(layers[-1], train=train))
with tf.variable_scope("layer4"):
layers.append(linear(layers[-1], 128, sn_op=None))
layers.append(lrelu(layers[-1]))
layers.append(batch_norm()(layers[-1], train=train))
with tf.variable_scope("layer5"):
layers.append(linear(layers[-1], self.output_length))
return layers[-1], layers
def build(self, image, train):
with tf.variable_scope(self.scope_name, reuse=tf.AUTO_REUSE):
layers = [image]
with tf.variable_scope("layer0"):
layers.append(
conv2d(layers[-1],
self.start_depth,
d_h=self.stride,
d_w=self.stride,
k_h=self.kernel,
k_w=self.kernel))
layers.append(lrelu(layers[-1]))
with tf.variable_scope("layer1"):
layers.append(
conv2d(layers[-1],
self.start_depth * 2,
d_h=self.stride,
d_w=self.stride,
k_h=self.kernel,
k_w=self.kernel))
layers.append(batch_norm()(layers[-1], train=train))
layers.append(lrelu(layers[-1]))
with tf.variable_scope("layer2"):
layers.append(
conv2d(layers[-1],
self.start_depth * 4,
d_h=self.stride,
d_w=self.stride,
k_h=self.kernel,
k_w=self.kernel))
layers.append(batch_norm()(layers[-1], train=train))
layers.append(lrelu(layers[-1]))
with tf.variable_scope("layer3"):
layers.append(
conv2d(layers[-1],
self.start_depth * 8,
d_h=self.stride,
d_w=self.stride,
k_h=self.kernel,
k_w=self.kernel))
layers.append(batch_norm()(layers[-1], train=train))
layers.append(lrelu(layers[-1]))
with tf.variable_scope("layer4"):
layers.append(linear(layers[-1], self.output_length))
return layers[-1], layers
def build(self, image, train):
with tf.variable_scope(self.scope_name, reuse=tf.AUTO_REUSE):
layers = [image]
with tf.variable_scope("layer0"):
layers.append(conv2d(
layers[-1],
32,
d_h=self.stride,
d_w=self.stride,
k_h=self.kernel,
k_w=self.kernel))
layers.append(tf.nn.relu(layers[-1]))
with tf.variable_scope("layer1"):
layers.append(conv2d(
layers[-1],
32,
d_h=self.stride,
d_w=self.stride,
k_h=self.kernel,
k_w=self.kernel))
layers.append(tf.nn.relu(layers[-1]))
with tf.variable_scope("layer2"):
layers.append(conv2d(
layers[-1],
64,
d_h=self.stride,
d_w=self.stride,
k_h=self.kernel,
k_w=self.kernel))
layers.append(tf.nn.relu(layers[-1]))
with tf.variable_scope("layer3"):
layers.append(conv2d(
layers[-1],
64,
d_h=self.stride,
d_w=self.stride,
k_h=self.kernel,
k_w=self.kernel))
layers.append(tf.nn.relu(layers[-1]))
with tf.variable_scope("layer4"):
layers.append(linear(layers[-1], 128))
with tf.variable_scope("layer5-mean"):
mean = linear(layers[-1], self.output_length)
with tf.variable_scope("layer5-logvar"):
logvar = linear(layers[-1], self.output_length)
layers.append(mean)
layers.append(logvar)
return mean, logvar, layers
def dense_trans(
self,
feats,
scope,
transition_rate: float = 0.5,
):
with tf.variable_scope(scope):
out_dim = int(int(feats.shape[-1]) * transition_rate)
feats = op.conv2d(feats, out_dim, 1, activation_fn=None)
feats = tf.nn.max_pool(feats, [1, 2, 2, 1], [1, 2, 2, 1], 'VALID')
return feats
def dense_block(self,
feats,
scope,
num_layers: int = 8,
growth_rate: int = 20,
kernel_size: int = 3):
with tf.variable_scope(scope):
for i in range(num_layers):
new_feats = op.conv2d(feats,
growth_rate, (kernel_size, kernel_size),
scope='conv2d-%d' % i)
feats = tf.concat([feats, new_feats], 3)
return feats
def build(self, image, train, sn_op):
with tf.variable_scope(self.scope_name, reuse=tf.AUTO_REUSE):
with tf.variable_scope("shared"):
layers = [image]
with tf.variable_scope("layer0"):
layers.append(
conv2d(layers[-1],
self.start_depth,
d_h=self.stride,
d_w=self.stride,
k_h=self.kernel,
k_w=self.kernel,
sn_op=sn_op))
layers.append(lrelu(layers[-1]))
with tf.variable_scope("layer1"):
layers.append(
conv2d(layers[-1],
self.start_depth * 2,
d_h=self.stride,
d_w=self.stride,
k_h=self.kernel,
k_w=self.kernel,
sn_op=sn_op))
layers.append(lrelu(layers[-1]))
with tf.variable_scope("layer2"):
layers.append(
conv2d(layers[-1],
self.start_depth * 4,
d_h=self.stride,
d_w=self.stride,
k_h=self.kernel,
k_w=self.kernel,
sn_op=sn_op))
layers.append(lrelu(layers[-1]))
with tf.variable_scope("layer3"):
layers.append(
conv2d(layers[-1],
self.start_depth * 8,
d_h=self.stride,
d_w=self.stride,
k_h=self.kernel,
k_w=self.kernel,
sn_op=sn_op))
layers.append(lrelu(layers[-1]))
with tf.variable_scope("x"):
image_layers = [layers[-1]]
with tf.variable_scope("layer0"):
image_layers.append(
linear(image_layers[-1], 1, sn_op=sn_op))
image_layers.append(tf.nn.sigmoid(image_layers[-1]))
with tf.variable_scope("q"):
q_layers = [layers[-1]]
with tf.variable_scope("layer0"):
q_layers.append(
linear(q_layers[-1], self.output_length, sn_op=sn_op))
layers.extend(image_layers)
layers.extend(q_layers)
return image_layers[-1], q_layers[-1], layers
def __init__(
self,
embeddings: embed.IndexedWordEmbedding,
class_num: int,
scale_l1: float = 0.0,
scale_l2: float = 0.000001,
encode_dim: int = 300,
fact_intr_dim: int = 10,
fact_proj_dim: int = -1,
char_filer_width: int = 5,
char_embed_dim: int = 8,
char_conv_dim: int = 100,
lstm_unit: int = 300,
) -> None:
super(DIIN, self).__init__()
self._class_num = class_num
self.scale_l1 = scale_l1
self.scale_l2 = scale_l2
self.encode_dim = encode_dim
self.fact_proj_dim = fact_proj_dim
self.fact_intr_dim = fact_intr_dim
self.char_filter_width = char_filer_width
self.char_embed_dim = char_embed_dim
self.char_conv_dim = char_conv_dim
self.lstm_unit = lstm_unit
self.keep_prob = tf.placeholder(tf.float32, shape=[])
batch_size = tf.shape(self.x1)[0]
padded_len1 = tf.shape(self.x1)[1]
padded_len2 = tf.shape(self.x2)[1]
op_kwargs = {
'scale_l1': self.scale_l1,
'scale_l2': self.scale_l2,
'keep_prob': self.keep_prob
}
with tf.variable_scope('embed') as s:
# Word pretrained embeddings (300D)
word_embed = tf.constant(embeddings.get_embeddings(),
dtype=tf.float32,
name='word_embed')
word_embed1, word_embed2 = map(lambda x: tf.gather(word_embed, x),
[self.x1, self.x2])
# Character convolutional embeddings (`char_conv_dim`D)
char_embed = op.get_variable('char_embed',
shape=(256, char_embed_dim))
char_filter = op.get_variable('char_filter',
shape=(1, self.char_filter_width,
self.char_embed_dim,
self.char_conv_dim))
def embed_chars(x_char):
embed = tf.gather(char_embed, x_char)
# shape: [batch, seq_len, word_len, embed_dim]
conv = tf.nn.conv2d(embed, char_filter, [1, 1, 1, 1], 'VALID')
# shape: [batch, seq_len, word_len - filter_width + 1, conv_dim]
return tf.reduce_max(conv, 2)
# shape: [batch, seq_len, conv_dim]
char_embed1, char_embed2 = map(embed_chars,
[self.char1, self.char2])
# Tag one-hot embeddings (72D)
def embed_tags(x_ids, x_tags, x_len):
x_tags *= tf.sequence_mask(x_len,
tf.shape(x_tags)[1],
dtype=tf.int32)
# shape: [batch, seq_len]
tag_embed = tf.one_hot(x_tags,
data.SNLI.TAGS,
dtype=tf.float32,
name='char_embed')
return tag_embed[:, :tf.shape(x_ids)[1]]
tag_embed1, tag_embed2 = map(
embed_tags,
*zip((self.x1, self.tag1, self.len1),
(self.x2, self.tag2, self.len2)))
# Merge embeddings
x1 = tf.concat([word_embed1, char_embed1, tag_embed1], 2)
x2 = tf.concat([word_embed2, char_embed2, tag_embed2], 2)
with tf.variable_scope('encode') as s:
# Highway encoding
def highway_encode(x):
x = op.highway(x,
scope='hw-1',
dim=self.encode_dim,
**op_kwargs)
x = op.highway(x,
scope='hw-2',
dim=self.encode_dim,
**op_kwargs)
return x
x1, x2 = map(highway_encode, [x1, x2])
# shape: [batch, seq_len, encode_dim]
# Self-attention
def self_attent(x, padded_len):
t1 = tf.tile(tf.expand_dims(x, 2), [1, 1, tf.shape(x)[1], 1])
t2 = tf.tile(tf.expand_dims(x, 1), [1, tf.shape(x)[1], 1, 1])
# shape: [batch, seq_len, seq_len, encode_dim]
t = tf.reshape(
tf.concat([t1, t2, t1 * t2], 3),
[batch_size, padded_len**2, 3 * self.encode_dim])
# shape: [batch, seq_len^2, 3 * encode_dim]
att = op.linear(t, dim=1, bias=None, activation_fn=None)
# shape: [batch, seq_len^2, 1]
att = tf.reshape(att, [batch_size, padded_len, padded_len])
# shape: [batch, seq_len, seq_len]
soft_align = tf.einsum('bik,bkj->bij', tf.nn.softmax(att), x)
return op.gated_fuse(x, soft_align)
# shape: [batch, seq_len, encode_dim]
x1, x2 = map(lambda x, l: self_attent(x, l),
*zip((x1, padded_len1), (x2, padded_len2)))
with tf.variable_scope('interact') as s:
inter = tf.expand_dims(x1, 2) * tf.expand_dims(x2, 1)
# shape: [batch, seq_len1, seq_len2, encode_dim]
with tf.variable_scope('extract') as s:
# Dense Net
feats = op.conv2d(inter, self.encode_dim * 0.3, 1)
# shape: [batch, seq_len1, seq_len2, encode_dim]
feats = self.dense_block(feats, 'dense-block-1')
feats = self.dense_trans(feats, 'dense-trans-1')
feats = self.dense_block(feats, 'dense-block-2')
feats = self.dense_trans(feats, 'dense-trans-2')
feats = self.dense_block(feats, 'dense-block-3')
feats = self.dense_trans(feats, 'dense-trans-3')
shape = tf.shape(feats)
feats = tf.reshape(feats,
[shape[0], shape[1] * shape[2] * shape[3]])
self.evaluate_and_loss(feats)
def __call__(self,
image,
reuse_private=False,
reuse_shared=False,
shared='shared',
private='private_task'):
layer_index = 0
residual_index = 0
with tf.variable_scope(self.module_name):
# First few layers in the classifier
with tf.variable_scope(private, reuse=reuse_private):
#print(image.get_shape().as_list())
# image_size - fiter_size + 2*pad + 1 (when stride=1)
x = tf.pad(image, [[0, 0], [3, 3], [3, 3], [0, 0]],
'SYMMETRIC')
x = op.conv2d(x,
out_channel=self.channel,
filter_size=7,
stride=1,
activation=tf.nn.relu,
padding='VALID',
normalization=op._batch_norm,
name='conv2d_%d' % layer_index,
training=self.training)
layer_index += 1
x, layer_index = op.dilated_residual_block(
x,
out_dim=self.channel,
layer_index=layer_index,
downsample=False,
normalization=op._batch_norm,
name='residual_%d' % residual_index,
training=self.training)
residual_index += 1
# Last layers in the classifier
with tf.variable_scope(shared, reuse=reuse_shared):
# Down sample 2
x, layer_index = op.dilated_residual_block(
x,
out_dim=self.channel * 2,
layer_index=layer_index,
downsample=True,
normalization=op._batch_norm,
name='residual_%d' % residual_index,
training=self.training)
residual_index += 1
x, layer_index = op.dilated_residual_block(
x,
out_dim=self.channel * 2,
layer_index=layer_index,
downsample=False,
normalization=op._batch_norm,
name='residual_%d' % residual_index,
training=self.training)
residual_index += 1
# Dilated convolution instead of down sample
x, layer_index = op.dilated_residual_block(
x,
out_dim=self.channel * 4,
dilation_rate=2,
layer_index=layer_index,
downsample=False,
normalization=op._batch_norm,
name='residual_%d' % residual_index,
training=self.training)
residual_index += 1
x, layer_index = op.dilated_residual_block(
x,
out_dim=self.channel * 4,
dilation_rate=2,
layer_index=layer_index,
downsample=False,
normalization=op._batch_norm,
name='residual_%d' % residual_index,
training=self.training)
residual_index += 1
# Removing gridding artifacts
x = op.conv2d(x,
out_channel=self.channel * 8,
filter_size=3,
stride=1,
activation=tf.nn.relu,
normalization=op._batch_norm,
name='conv2d_%d' % layer_index,
training=self.training)
layer_index += 1
x = op.conv2d(x,
out_channel=self.channel * 8,
filter_size=3,
stride=1,
activation=tf.nn.relu,
normalization=op._batch_norm,
name='conv2d_%d' % layer_index,
training=self.training)
layer_index += 1
# Global average pooling for classification output
x = op.global_average_pooling(x)
head_logits = op.fc(x,
self.num_classes,
dropout=False,
activation=None,
name='head')
lateral_logits = op.fc(x,
self.num_classes,
dropout=False,
activation=None,
name='latera')
return head_logits, lateral_logits
def __call__(self,
image,
measurements,
reuse_private=False,
reuse_shared=False,
private='private',
shared='shared_task',
reuse=False):
image_layer_index = 0
fc_index = 0
residual_index = 0
branches = list()
branches_feature = list()
with tf.variable_scope(self.module_name):
with tf.variable_scope(self.name):
if reuse:
tf.get_variable_scope().reuse_variables()
with tf.variable_scope('image_module'):
with tf.variable_scope(private, reuse=reuse_private):
# [96, 96]
x = op.conv2d(image,
out_channel=self.channel,
filter_size=7,
stride=2,
normalization=op._group_norm,
activation=tf.nn.relu,
name='conv2d_%d' % image_layer_index)
image_layer_index += 1
with tf.variable_scope(shared, reuse=reuse_shared):
# [48, 48]
x, image_layer_index = op.residual_block(
x,
out_dim=self.channel,
layer_index=image_layer_index,
normalization=op._group_norm,
downsample=False,
training=self.training)
x, image_layer_index = op.residual_block(
x,
out_dim=self.channel,
layer_index=image_layer_index,
normalization=op._group_norm,
downsample=False,
training=self.training)
x, image_layer_index = op.residual_block(
x,
out_dim=self.channel * 2,
layer_index=image_layer_index,
normalization=op._group_norm,
downsample=True,
training=self.training)
x, image_layer_index = op.residual_block(
x,
out_dim=self.channel * 2,
layer_index=image_layer_index,
normalization=op._group_norm,
downsample=False,
training=self.training)
if self.classifier_type == 'DRN':
# Dilated convolution instead of down sample
x, image_layer_index = op.dilated_residual_block(
x,
out_dim=self.channel * 4,
dilation_rate=2,
layer_index=image_layer_index,
downsample=False,
normalization=op._group_norm,
name='residual_%d' % residual_index,
training=self.training)
residual_index += 1
x, image_layer_index = op.dilated_residual_block(
x,
out_dim=self.channel * 4,
dilation_rate=2,
layer_index=image_layer_index,
downsample=False,
normalization=op._group_norm,
name='residual_%d' % residual_index,
training=self.training)
residual_index += 1
x, image_layer_index = op.dilated_residual_block(
x,
out_dim=self.channel * 8,
dilation_rate=4,
layer_index=image_layer_index,
downsample=False,
normalization=op._group_norm,
name='residual_%d' % residual_index,
training=self.training)
residual_index += 1
x, image_layer_index = op.dilated_residual_block(
x,
out_dim=self.channel * 8,
dilation_rate=4,
layer_index=image_layer_index,
downsample=False,
normalization=op._group_norm,
name='residual_%d' % residual_index,
training=self.training)
residual_index += 1
# Removing gridding artifacts
x = op.dilated_conv2d(x,
out_channel=self.channel * 8,
filter_size=3,
dilation_rate=2,
activation=tf.nn.relu,
normalization=op._group_norm,
padding='SAME',
name='conv2d_%d' %
image_layer_index,
training=self.training)
image_layer_index += 1
x = op.dilated_conv2d(x,
out_channel=self.channel * 8,
filter_size=3,
activation=tf.nn.relu,
dilation_rate=2,
normalization=op._group_norm,
padding='SAME',
name='conv2d_%d' %
image_layer_index,
training=self.training)
image_layer_index += 1
x = op.conv2d(x,
out_channel=self.channel * 8,
filter_size=3,
stride=1,
activation=tf.nn.relu,
normalization=op._group_norm,
name='conv2d_%d' % image_layer_index,
training=self.training)
image_layer_index += 1
x = op.conv2d(x,
out_channel=self.channel * 8,
filter_size=3,
stride=1,
activation=tf.nn.relu,
normalization=op._group_norm,
name='conv2d_%d' % image_layer_index,
training=self.training)
image_layer_index += 1
elif self.classifier_type == 'RESNET':
# [24, 24]
x, image_layer_index = op.residual_block(
x,
out_dim=self.channel * 4,
layer_index=image_layer_index,
normalization=op._group_norm,
downsample=True,
training=self.training)
x, image_layer_index = op.residual_block(
x,
out_dim=self.channel * 4,
layer_index=image_layer_index,
normalization=op._group_norm,
downsample=False,
training=self.training)
# [12, 12]
x, image_layer_index = op.residual_block(
x,
out_dim=self.channel * 8,
layer_index=image_layer_index,
normalization=op._group_norm,
downsample=True,
training=self.training)
x, image_layer_index = op.residual_block(
x,
out_dim=self.channel * 8,
layer_index=image_layer_index,
normalization=op._group_norm,
downsample=False,
training=self.training)
x = op.global_average_pooling(x)
#with tf.variable_scope('measurement_module'):
# y = op.fc(measurements, self.measurement_fc, dropout_ratio=self.dropout[fc_index], name='fc_%d'%fc_index)
# fc_index += 1
# y = op.fc(y, self.measurement_fc, dropout_ratio=self.dropout[fc_index], name='fc_%d'%fc_index)
# fc_index += 1
with tf.variable_scope('joint'):
#joint = tf.concat([x,y], axis=-1, name='joint_representation')
joint = x
for i in range(self.num_commands):
with tf.variable_scope('branch_%d' % i):
branch_output = op.fc(joint,
5,
dropout=False,
activation=None,
name='fc_%d' % fc_index)
branches.append(branch_output)
return branches, x
def __call__(self,
x,
name,
patch=False,
reuse=False,
dropout_prob=0.7,
training=True):
layer_index = 0
activations = list()
if training:
self.dropout_prob = dropout_prob
else:
self.dropout_prob = 1.0
with tf.variable_scope(self.module_name):
with tf.variable_scope(name):
# Add optional noise
def add_noise(hidden, scope_num=None):
if scope_num:
hidden = slim.dropout(hidden,
self.dropout_prob,
is_training=training,
scope='dropout_%s' % scope_num)
return hidden + tf.random_normal(
hidden.shape.as_list(), mean=0, stddev=0.1)
if reuse:
tf.get_variable_scope().reuse_variables()
# From cycleGAN, do not use instance Norm for the first C64 layer
# 256,256
x = op.conv2d(x,
out_channel=self.channel,
stride=1,
normalization=False,
name='conv2d_%d' % layer_index)
x = add_noise(x, layer_index)
layer_index += 1
# 256,256
x = op.conv2d(x,
out_channel=self.channel * 2,
name='conv2d_%d' % layer_index)
activations.append(x)
x = add_noise(x, layer_index)
layer_index += 1
# 128,128
x = op.conv2d(x,
out_channel=self.channel * 4,
name='conv2d_%d' % layer_index)
activations.append(x)
x = add_noise(x, layer_index)
layer_index += 1
# 64,64
x = op.conv2d(x,
out_channel=self.channel * 8,
name='conv2d_%d' % layer_index)
activations.append(x)
x = add_noise(x, layer_index)
layer_index += 1
# 32,32
x = op.conv2d(x,
out_channel=self.channel * 16,
name='conv2d_%d' % layer_index)
activations.append(x)
x = add_noise(x, layer_index)
layer_index += 1
if self.group_size > 1:
x = self.minibatch_discrimination(x, self.group_size)
#activations.append(x)
#layer_index += 1
#x = op._max_pool(x)
if patch:
tf.logging.info('Patch GAN for %s' % name)
# After the last layer, a convolution is applied to map to a 1 dimensional output
x = op.conv2d(x,
out_channel=1,
stride=1,
name='conv2d_%d' % layer_index,
activation=None,
normalization=False)
else:
tf.logging.info('No Patch GAN for %s' % name)
x = slim.flatten(x)
x = op.fc(x, 1, activation=None, dropout=False, name='fc')
# activations acter RELU
return x, activations
def generator_unet(self, x, name, att=False, reuse=False):
layer_index = 0
normalize_func = self.normalize()
with tf.variable_scope(self.module_name):
with tf.variable_scope(name + '_UNET'):
if reuse:
tf.get_variable_scope().reuse_variables()
if self.latent_vars:
noise_channel = project_latent_vars(
shape=x.get_shape().as_list()[0:3] + [1],
latent_vars=self.latent_vars,
combine_method='concat',
name=name)
x = tf.concat([x, noise_channel], axis=3)
with tf.variable_scope('encoder'):
e1 = op.conv2d(x,
out_channel=self.channel,
name='conv2d_%d' % layer_index,
normalization=False,
activation=False)
# 128,128
layer_index += 1
e2 = op.conv2d(tf.nn.relu(e1),
out_channel=self.channel * 2,
name='conv2d_%d' % layer_index,
activation=False,
normalization=normalize_func)
# 64,64
layer_index += 1
e3 = op.conv2d(tf.nn.relu(e2),
out_channel=self.channel * 4,
name='conv2d_%d' % layer_index,
activation=False,
normalization=normalize_func)
# 32,32
layer_index += 1
e4 = op.conv2d(tf.nn.relu(e3),
out_channel=self.channel * 8,
name='conv2d_%d' % layer_index,
activation=False,
normalization=normalize_func)
# 16,16
layer_index += 1
e5 = op.conv2d(tf.nn.relu(e4),
out_channel=self.channel * 8,
name='conv2d_%d' % layer_index,
activation=False,
normalization=normalize_func)
# 8,8
layer_index += 1
e6 = op.conv2d(tf.nn.relu(e5),
out_channel=self.channel * 8,
name='conv2d_%d' % layer_index,
activation=False,
normalization=normalize_func)
# 4,4
layer_index += 1
e7 = op.conv2d(tf.nn.relu(e6),
out_channel=self.channel * 8,
name='conv2d_%d' % layer_index,
activation=False,
normalization=normalize_func)
# 2,2
layer_index += 1
# Middle point of total architecture(number of total layers=16)
e8 = op.conv2d(tf.nn.relu(e7),
out_channel=self.channel * 8,
name='conv2d_%d' % layer_index,
activation=False,
normalization=normalize_func)
# 1,1
layer_index += 1
# U-Net architecture is with skip connections between each layer i in the encoer and layer n-i in the decoder. Concatenate activations in channel axis
# Dropout with 0.5
with tf.variable_scope('decoder'):
d1 = op.transpose_conv2d(tf.nn.relu(e8),
out_channel=self.channel * 8,
name='transpose_conv2d_%d' %
layer_index,
activation=False,
dropout=True,
normalization=normalize_func)
d1 = tf.concat([d1, e7], axis=3)
# 2,2
layer_index += 1
d2 = op.transpose_conv2d(tf.nn.relu(d1),
out_channel=self.channel * 8,
name='transpose_conv2d_%d' %
layer_index,
activation=False,
dropout=True,
normalization=normalize_func)
d2 = tf.concat([d2, e6], axis=3)
# 4,4
layer_index += 1
d3 = op.transpose_conv2d(tf.nn.relu(d2),
out_channel=self.channel * 8,
name='transpose_conv2d_%d' %
layer_index,
activation=False,
dropout=True,
normalization=normalize_func)
d3 = tf.concat([d3, e5], axis=3)
# 8,8
layer_index += 1
d4 = op.transpose_conv2d(tf.nn.relu(d3),
out_channel=self.channel * 8,
name='transpose_conv2d_%d' %
layer_index,
activation=False,
normalization=normalize_func)
d4 = tf.concat([d4, e4], axis=3)
# 16,16
layer_index += 1
if att:
d5 = op.transpose_conv2d(tf.nn.relu(d4),
out_channel=self.channel * 4,
name='transpose_conv2d_%d' %
layer_index,
activation=False,
normalization=normalize_func)
e3 = op.attention_gate(e3,
d5,
self.channel * 4,
layer_index,
normalization=normalize_func)
d5 = tf.concat([d5, e3], axis=3)
# 32,32
layer_index += 1
d6 = op.transpose_conv2d(tf.nn.relu(d5),
out_channel=self.channel * 2,
name='transpose_conv2d_%d' %
layer_index,
activation=False,
normalization=normalize_func)
e2 = op.attention_gate(e2,
d6,
self.channel * 2,
layer_index,
normalization=normalize_func)
d6 = tf.concat([d6, e2], axis=3)
# 64,64
layer_index += 1
d7 = op.transpose_conv2d(tf.nn.relu(d6),
out_channel=self.channel,
name='transpose_conv2d_%d' %
layer_index,
activation=False,
normalization=normalize_func)
e1 = op.attention_gate(e1,
d7,
self.channel,
layer_index,
normalization=normalize_func)
d7 = tf.concat([d7, e1], axis=3)
# 128,128
layer_index += 1
d8 = op.transpose_conv2d(tf.nn.relu(d7),
out_channel=self.out_channel,
name='transpose_conv2d_%d' %
layer_index,
normalization=False,
activation=tf.nn.tanh)
# 256,256
else:
tf.logging.info('No attention for %s' % name)
d5 = op.transpose_conv2d(tf.nn.relu(d4),
out_channel=self.channel * 4,
name='transpose_conv2d_%d' %
layer_index,
activation=False,
normalization=normalize_func)
d5 = tf.concat([d5, e3], axis=3)
# 32,32
layer_index += 1
d6 = op.transpose_conv2d(tf.nn.relu(d5),
out_channel=self.channel * 2,
name='transpose_conv2d_%d' %
layer_index,
activation=False,
normalization=normalize_func)
d6 = tf.concat([d6, e2], axis=3)
# 64,64
layer_index += 1
d7 = op.transpose_conv2d(tf.nn.relu(d6),
out_channel=self.channel,
name='transpose_conv2d_%d' %
layer_index,
activation=False,
normalization=normalize_func)
d7 = tf.concat([d7, e1], axis=3)
# 128,128
layer_index += 1
d8 = op.transpose_conv2d(tf.nn.relu(d7),
out_channel=self.out_channel,
name='transpose_conv2d_%d' %
layer_index,
normalization=False,
activation=tf.nn.tanh)
# 256,256
return d8, noise_channel
def generator_resnet(self, x, name, reuse=False):
layer_index = 0
residual_index = 0
with tf.variable_scope(self.module_name):
with tf.variable_scope(name + '_RESNET'):
if reuse:
tf.get_variable_scope().reuse_variables()
if self.latent_vars:
noise_channel = project_latent_vars(
shape=x.get_shape().as_list()[0:3] + [1],
latent_vars=self.latent_vars,
combine_method='concat',
name=name)
x = tf.concat([x, noise_channel], axis=3)
# image_size - fiter_size + 2*pad + 1 (when stride=1)
x = tf.pad(x, [[0, 0], [3, 3], [3, 3], [0, 0]], 'REFLECT')
x = op.conv2d(x,
out_channel=self.channel,
filter_size=7,
stride=1,
activation=tf.nn.relu,
padding='VALID',
name='conv2d_%d' % layer_index,
normalization=False)
layer_index += 1
x = op.conv2d(x,
out_channel=self.channel * 2,
filter_size=3,
stride=2,
activation=tf.nn.relu,
name='conv2d_%d' % layer_index)
layer_index += 1
x = op.conv2d(x,
out_channel=self.channel * 4,
filter_size=3,
stride=2,
activation=tf.nn.relu,
name='conv2d_%d' % layer_index)
layer_index += 1
x, layer_index = op.residual_block(x,
out_dim=self.channel * 4,
layer_index=layer_index,
name='residual_%d' %
residual_index)
layer_index += 1
x, layer_index = op.residual_block(x,
out_dim=self.channel * 4,
layer_index=layer_index,
name='residual_%d' %
residual_index)
residual_index += 1
x, layer_index = op.residual_block(x,
out_dim=self.channel * 4,
layer_index=layer_index,
name='residual_%d' %
residual_index)
residual_index += 1
x, layer_index = op.residual_block(x,
out_dim=self.channel * 4,
layer_index=layer_index,
name='residual_%d' %
residual_index)
residual_index += 1
x, layer_index = op.residual_block(x,
out_dim=self.channel * 4,
layer_index=layer_index,
name='residual_%d' %
residual_index)
residual_index += 1
x, layer_index = op.residual_block(x,
out_dim=self.channel * 4,
layer_index=layer_index,
name='residual_%d' %
residual_index)
residual_index += 1
# x, layer_index = op.residual_block(x, out_dim=self.channel*4, layer_index=layer_index, name='residual_%d'%residual_index)
#residual_index += 1
#x, layer_index = op.residual_block(x, out_dim=self.channel*4, layer_index=layer_index, name='residual_%d'%residual_index)
#residual_index += 1
#x, layer_index = op.residual_block(x, out_dim=self.channel*4, layer_index=layer_index, name='residual_%d'%residual_index)
#residual_index += 1
# Upsampling
x = op.transpose_conv2d(x,
out_channel=self.channel * 2,
filter_size=3,
stride=2,
name='transpose_conv2d_%d' %
layer_index)
layer_index += 1
x = op.transpose_conv2d(x,
out_channel=self.channel,
filter_size=3,
stride=2,
name='transpose_conv2d_%d' %
layer_index)
layer_index += 1
#x = tf.pad(x, [[0,0],[3,3],[3,3],[0,0]], 'REFLECT')
#x = op.conv2d(x, out_channel=3, filter_size=7, stride=1, padding='VALID', name='conv2d_%d'%layer_index, normalization=None, activation=tf.nn.tanh)
#x = op.conv2d(x, out_channel=3, filter_size=1, stride=1, padding='SAME', name='conv2d_%d'%layer_index, normalization=None, activation=tf.nn.tanh)
x = op.transpose_conv2d(x,
out_channel=self.out_channel,
filter_size=7,
stride=1,
name='transpose_conv2d_%d' %
layer_index,
normalization=False,
activation=tf.nn.tanh)
return x, noise_channel
def generator_drn(self, x, name, reuse=False):
layer_index = 0
residual_index = 0
normalize_func = self.normalize()
with tf.variable_scope(self.module_name):
with tf.variable_scope(name + '_DRN'):
if reuse:
tf.get_variable_scope().reuse_variables()
if self.latent_vars:
noise_channel = project_latent_vars(
shape=x.get_shape().as_list()[0:3] + [1],
latent_vars=self.latent_vars,
combine_method='concat',
name=name)
x = tf.concat([x, noise_channel], axis=3)
# image_size - fiter_size + 2*pad + 1 (when stride=1)
x = tf.pad(x, [[0, 0], [3, 3], [3, 3], [0, 0]], 'REFLECT')
x = op.conv2d(x,
out_channel=self.channel,
filter_size=7,
stride=1,
activation=tf.nn.relu,
padding='VALID',
name='conv2d_%d' % layer_index,
normalization=False)
layer_index += 1
x, layer_index = op.dilated_residual_block(
x,
out_dim=self.channel,
layer_index=layer_index,
downsample=False,
name='residual_%d' % residual_index,
normalization=normalize_func)
layer_index += 1
# Down sample
x, layer_index = op.dilated_residual_block(
x,
out_dim=self.channel * 2,
layer_index=layer_index,
downsample=True,
name='residual_%d' % residual_index,
normalization=normalize_func)
residual_index += 1
x, layer_index = op.dilated_residual_block(
x,
out_dim=self.channel * 2,
layer_index=layer_index,
downsample=False,
name='residual_%d' % residual_index,
normalization=normalize_func)
residual_index += 1
# Dilation instead of down sample
x, layer_index = op.dilated_residual_block(
x,
out_dim=self.channel * 4,
dilation_rate=2,
layer_index=layer_index,
downsample=False,
name='residual_%d' % residual_index,
normalization=normalize_func)
residual_index += 1
x, layer_index = op.dilated_residual_block(
x,
out_dim=self.channel * 4,
dilation_rate=2,
layer_index=layer_index,
downsample=False,
name='residual_%d' % residual_index,
normalization=normalize_func)
residual_index += 1
# x = op.dilated_conv2d(x, out_channel=self.channel*4, filter_size=3, activation=tf.nn.relu, dilation_rate=2, name='conv2d_%d'%layer_index, padding='SAME')
# layer_index += 1
# x = op.dilated_conv2d(x, out_channel=self.channel*4, filter_size=3, activation=tf.nn.relu, dilation_rate=2, name='conv2d_%d'%layer_index, padding='SAME')
# layer_index += 1
# Removing gridding artifacts
x = op.conv2d(x,
out_channel=self.channel * 2,
filter_size=3,
stride=1,
activation=tf.nn.relu,
name='conv2d_%d' % layer_index,
normalization=normalize_func)
layer_index += 1
x = op.conv2d(x,
out_channel=self.channel * 2,
filter_size=3,
stride=1,
activation=tf.nn.relu,
name='conv2d_%d' % layer_index,
normalization=normalize_func)
layer_index += 1
# Upsampling
x = op.transpose_conv2d(x,
out_channel=self.channel,
filter_size=3,
name='transpose_conv2d_%d' %
layer_index,
normalization=normalize_func)
layer_index += 1
#x = tf.pad(x, [[0,0],[3,3],[3,3],[0,0]], 'REFLECT')
#x = op.conv2d(x, out_channel=3, filter_size=7, stride=1, padding='VALID', name='transpose_conv2d_%d'%layer_index, normalization=None, activation=tf.nn.tanh)
x = op.transpose_conv2d(x,
out_channel=self.out_channel,
filter_size=7,
stride=1,
name='transpose_conv2d_%d' %
layer_index,
normalization=None,
activation=tf.nn.tanh)
return x, noise_channel