def _encode(self):
with tf.variable_scope("passage_question_encoding"):
self.p_encode = residual_block(self.p_emb,
num_blocks=1,
num_conv_layers=4,
kernel_size=7,
mask=None,
num_filters=self.hidden_size,
num_heads=self.num_head,
seq_len=self.p_length,
scope="Encoder_Residual_Block",
bias=False,
dropout=self.dropout)
self.q_encode = residual_block(
self.q_emb,
num_blocks=1,
num_conv_layers=4,
kernel_size=7,
mask=None,
num_filters=self.hidden_size,
num_heads=self.num_head,
seq_len=self.q_length,
scope="Encoder_Residual_Block",
reuse=True, # Share the weights between passage and question
bias=False,
dropout=self.dropout)
def get_evaluate_model(width, height):
input_o = layers.Input(shape=(height, width, 3),
dtype='float32',
name='input_o')
c1 = layers.Conv2D(32, (9, 9), strides=1, padding='same',
name='conv_1')(input_o)
c1 = layers.BatchNormalization(name='normal_1')(c1)
c1 = layers.Activation('relu', name='relu_1')(c1)
c2 = layers.Conv2D(64, (3, 3), strides=2, padding='same',
name='conv_2')(c1)
c2 = layers.BatchNormalization(name='normal_2')(c2)
c2 = layers.Activation('relu', name='relu_2')(c2)
c3 = layers.Conv2D(128, (3, 3), strides=2, padding='same',
name='conv_3')(c2)
c3 = layers.BatchNormalization(name='normal_3')(c3)
c3 = layers.Activation('relu', name='relu_3')(c3)
r1 = residual_block(c3, 1)
r2 = residual_block(r1, 2)
r3 = residual_block(r2, 3)
r4 = residual_block(r3, 4)
r5 = residual_block(r4, 5)
d1 = layers.Conv2DTranspose(64, (3, 3),
strides=2,
padding='same',
name='conv_4')(r5)
d1 = layers.BatchNormalization(name='normal_4')(d1)
d1 = layers.Activation('relu', name='relu_4')(d1)
d2 = layers.Conv2DTranspose(32, (3, 3),
strides=2,
padding='same',
name='conv_5')(d1)
d2 = layers.BatchNormalization(name='normal_5')(d2)
d2 = layers.Activation('relu', name='relu_5')(d2)
c4 = layers.Conv2D(3, (9, 9), strides=1, padding='same', name='conv_6')(d2)
c4 = layers.BatchNormalization(name='normal_6')(c4)
c4 = layers.Activation('tanh', name='tanh_1')(c4)
c4 = OutputScale(name='output')(c4)
model = Model([input_o], c4)
print("evaluate model built successfully!")
return model
def relation_block(self, processed_obs):
entities = build_entities(processed_obs, self.reduce_obs)
print('entities:', entities)
# [B,n_heads,N,Deepth=D+2]
MHDPA_output, self.relations = MHDPA(entities, n_heads=2)
print('MHDPA_output', MHDPA_output)
# [B,n_heads,N,Deepth]
residual_output = residual_block(entities, MHDPA_output)
print('residual_output', residual_output)
# max_pooling [B,n_heads,N,Deepth] --> [B,n_heads,Deepth]
maxpooling_output = tf.reduce_max(residual_output, axis=2)
print('maxpooling_output', maxpooling_output)
# [B,n_heads*Deepth]
# output = tf.layers.flatten(maxpooling_output)
# output = layerNorm(output, "relation_layerNorm")
# print('relation_layerNorm', output)
return maxpooling_output
def _fuse(self):
with tf.variable_scope("model_encoder_layer"):
inputs = tf.concat(self.attention_outputs, axis=-1)
self.enc = [
conv(inputs, self.hidden_size, name="input_projection")
]
for i in range(3):
if i % 2 == 0: # dropout every 2 blocks
self.enc[i] = tf.nn.dropout(self.enc[i],
1.0 - self.dropout)
self.enc.append(
residual_block(self.enc[i],
num_blocks=7,
num_conv_layers=2,
kernel_size=5,
mask=None,
num_filters=self.hidden_size,
num_heads=self.num_head,
seq_len=self.p_length,
scope="Model_Encoder",
bias=False,
reuse=True if i > 0 else None,
dropout=self.dropout))
def _build_network(self, input_placeholder):
x = input_placeholder
# Initial convolution to get to the correct number of filters:
x = tf.layers.conv3d(x,
self._params['N_INITIAL_FILTERS'],
kernel_size=[5, 5, 5],
strides=[1, 1, 1],
padding='same',
use_bias=False,
trainable=self._params['TRAINING'],
name="Conv2DInitial",
reuse=None)
# ReLU:
x = tf.nn.relu(x)
# Begin the process of residual blocks and downsampling:
for i in xrange(self._params['NETWORK_DEPTH']):
for j in xrange(self._params['RESIDUAL_BLOCKS_PER_LAYER']):
x = residual_block(
x,
self._params['TRAINING'],
batch_norm=self._params['BATCH_NORMALIZATION'],
name="resblock_down_{0}_{1}".format(i, j))
x = downsample_block(
x,
self._params['TRAINING'],
batch_norm=self._params['BATCH_NORMALIZATION'],
name="downsample_{0}".format(i))
# At the bottom, do another residual block:
for j in xrange(self._params['RESIDUAL_BLOCKS_PER_LAYER']):
x = residual_block(x,
self._params['TRAINING'],
batch_norm=self._params['BATCH_NORMALIZATION'],
name="deepest_block_{0}".format(j))
# At this point, we ought to have a network that has the same shape as the initial input, but with more filters.
# We can use a bottleneck to map it onto the right dimensions:
x = tf.layers.conv3d(
x,
self._params['NUM_LABELS'],
kernel_size=[1, 1, 1],
strides=[1, 1, 1],
padding='same',
activation=None,
use_bias=False,
trainable=self._params['TRAINING'],
name="BottleneckConv2D",
)
# Apply global average pooling to each filter to get the values for signal/bkg
# For global average pooling, need to get the shape of the input:
shape = (x.shape[1], x.shape[2], x.shape[3])
x = tf.nn.pool(x,
window_shape=shape,
pooling_type="AVG",
padding="VALID",
dilation_rate=None,
strides=None,
name="GlobalAveragePool",
data_format=None)
# Reshape to remove empty dimensions:
x = tf.reshape(x, [tf.shape(x)[0], self._params['NUM_LABELS']],
name="global_pooling_reshape")
# The final activation is softmax across the pixels. It gets applied in the loss function
# x = tf.nn.softmax(x)
return x
def _build_network(self, input_placeholder):
x = input_placeholder
# Initially, downsample by a factor of 2 to make the input data smaller:
x = tf.layers.average_pooling2d(x,
2,
2,
padding='same',
name="InitialAveragePooling")
# The filters are concatenated at some point, and progress together
if self._params['SHARE_PLANE_WEIGHTS']:
sharing = True
verbose = True
if verbose:
print "Initial shape: " + str(x.get_shape())
n_planes = self._params['NPLANES']
x = tf.split(x, n_planes * [1], -1)
if verbose:
for p in range(len(x)):
print "Plane {0} initial shape:".format(p) + str(
x[p].get_shape())
# Initial convolution to get to the correct number of filters:
for p in range(len(x)):
name = "Conv2DInitial"
if not sharing:
name += "_plane{0}".format(p)
# Only reuse on the non-first times through:
if p == 0:
reuse = False
else:
reuse = sharing
x[p] = tf.layers.conv2d(x[p],
self._params['N_INITIAL_FILTERS'],
kernel_size=[7, 7],
strides=[2, 2],
padding='same',
use_bias=False,
trainable=self._params['TRAINING'],
name=name,
reuse=reuse)
# ReLU:
x[p] = tf.nn.relu(x[p])
if verbose:
print "After initial convolution: "
for p in range(len(x)):
print "Plane {0}".format(p) + str(x[p].get_shape())
for p in xrange(len(x)):
name = "initial_resblock1"
if not sharing:
name += "_plane{0}".format(p)
# Only reuse on the non-first times through:
if p == 0:
reuse = False
else:
reuse = sharing
x[p] = residual_block(x[p],
self._params['TRAINING'],
batch_norm=True,
reuse=reuse,
name=name)
name = "initial_resblock2"
if not sharing:
name += "_plane{0}".format(p)
x[p] = residual_block(x[p],
self._params['TRAINING'],
batch_norm=True,
reuse=reuse,
name=name)
# Begin the process of residual blocks and downsampling:
for p in xrange(len(x)):
for i in xrange(self._params['NETWORK_DEPTH_PRE_MERGE']):
name = "downsample_{0}".format(i)
if not sharing:
name += "_plane{0}".format(p)
# Only reuse on the non-first times through:
if p == 0:
reuse = False
else:
reuse = sharing
x[p] = downsample_block(x[p],
self._params['TRAINING'],
batch_norm=True,
reuse=reuse,
name=name)
for j in xrange(self._params['RESIDUAL_BLOCKS_PER_LAYER']):
name = "resblock_{0}_{1}".format(i, j)
if not sharing:
name += "_plane{0}".format(p)
x[p] = residual_block(x[p],
self._params['TRAINING'],
batch_norm=True,
reuse=reuse,
name=name)
if verbose:
print "Plane {p}, layer {i}: x[{p}].get_shape(): {s}".format(
p=p, i=i, s=x[p].get_shape())
# Add a bottleneck to prevent the number of layers from exploding:
n_current_filters = x[p].get_shape().as_list()[-1]
if n_current_filters > self._params['N_MAX_FILTERS']:
n_filters = self._params['N_MAX_FILTERS']
else:
n_filters = n_current_filters
name = "Bottleneck_downsample_{0}".format(i)
if not sharing:
name += "_plane{0}".format(p)
x[p] = tf.layers.conv2d(x[p],
n_filters,
kernel_size=[1, 1],
strides=[1, 1],
padding='same',
activation=None,
use_bias=False,
trainable=self._params['TRAINING'],
reuse=reuse,
name=name)
# print "Reached the deepest layer."
# Here, concatenate all the planes together before the residual block:
x = tf.concat(x, axis=-1)
if verbose:
print "Shape after concatenation: " + str(x.get_shape())
# At the bottom, do another residual block:
for i in xrange(self._params['NETWORK_DEPTH_POST_MERGE']):
x = downsample_block(x,
self._params['TRAINING'],
batch_norm=True,
name="downsample_postmerge{0}".format(i))
for j in xrange(self._params['RESIDUAL_BLOCKS_PER_LAYER']):
x = residual_block(x,
self._params['TRAINING'],
batch_norm=True,
name="resblock_postmerge_{0}_{1}".format(
i, j))
# Apply bottlenecking here to keep the number of filters in check:
x = tf.layers.conv2d(
x,
self._params['N_MAX_FILTERS'],
kernel_size=[1, 1],
strides=[1, 1],
padding='same',
activation=None,
use_bias=False,
trainable=self._params['TRAINING'],
name="Bottleneck_downsample_merged_{0}".format(i))
if verbose:
print "Shape after final block: " + str(x.get_shape())
# Apply a bottle neck to get the right shape:
x = tf.layers.conv2d(x,
self._params['NUM_LABELS'],
kernel_size=[1, 1],
strides=[1, 1],
padding='same',
activation=None,
use_bias=False,
trainable=self._params['TRAINING'],
name="BottleneckConv2D")
if verbose:
print "Shape after bottleneck: " + str(x.get_shape())
# Apply global average pooling to get the right final shape:
shape = (x.shape[1], x.shape[2])
x = tf.nn.pool(x,
window_shape=shape,
pooling_type="AVG",
padding="VALID",
dilation_rate=None,
strides=None,
name="GlobalAveragePool",
data_format=None)
if verbose:
print "Shape after pooling: " + str(x.get_shape())
# Reshape to the right shape for logits:
x = tf.reshape(x, [tf.shape(x)[0], self._params['NUM_LABELS']],
name="global_pooling_reshape")
if verbose:
print "Final shape: " + str(x.get_shape())
return x
def build_generator(
input_tensor,
n_initial_filters=64,
n_blocks=2,
is_training=True,
reuse=False,
):
"""
Build a DC GAN generator with deep convolutional layers
Input_tensor is assumed to be reshaped into a (BATCH, L, H, F) type format (rectangular)
"""
with tf.variable_scope("generator"):
# Map the input to a small but many-filtered set up:
input_shape = input_tensor.get_shape()
print input_shape
# Want the output tensor to have a small number of spatial dimensions (7x7 for mnist)
# output size will be (W-F+2P)/S + 1
# input of 10, F=5, P = 0, S= 2 gives output size of (10 - 5)/2 + 1 = 3
# To get a specific outputsize (here == 7) with P == 1, and S = 2, set F as
# 7 = 1 + (10 - F)/1 -> 6 = 10 -F, or F = 4
x = tf.layers.conv2d(
input_tensor,
n_initial_filters,
kernel_size=[4, 4],
strides=[1, 1],
padding='valid',
activation=None,
use_bias=False,
kernel_initializer=None, # automatically uses Xavier initializer
kernel_regularizer=None,
activity_regularizer=None,
trainable=is_training,
name="Conv2D",
reuse=None)
print x.get_shape()
# Apply residual mappings and upsamplings:
for block in xrange(n_blocks):
x = residual_block(x,
is_training=is_training,
kernel=[3, 3],
stride=[1, 1],
alpha=0.0,
name="res_block_{}".format(block),
reuse=reuse)
x = upsample_block(x,
is_training=is_training,
kernel=[3, 3],
stride=[1, 1],
name="res_block_upsample_{}".format(block))
# Apply a 1x1 convolution to map to just one output filter:
x = tf.layers.conv2d(
x,
1,
kernel_size=[3, 3],
strides=[1, 1],
padding='same',
activation=None,
use_bias=False,
kernel_initializer=None, # automatically uses Xavier initializer
kernel_regularizer=None,
activity_regularizer=None,
trainable=is_training,
name="FinalConv2D1x1",
reuse=None)
# For the final activation, apply tanh:
return tf.nn.tanh(x)
def build_classifier(input_tensor,
n_output_classes=10,
is_training=True,
n_initial_filters=12,
initial_kernel=3,
initial_stride=1,
n_blocks=4,
downsample_interval=1):
with tf.variable_scope("mnist_classifier"):
# Initial convolutional layer:
x = tf.layers.conv2d(input_tensor,
n_initial_filters,
kernel_size=(initial_kernel,
initial_kernel),
strides=(initial_stride,
initial_stride),
padding='same',
activation=None,
use_bias=False,
bias_initializer=tf.zeros_initializer(),
trainable=is_training,
name="InitialConv2D",
reuse=None)
for i in xrange(n_blocks):
if i != 0 and i % downsample_interval == 0:
x = downsample_block(x, name="res_block_downsample_{}".format(i),
is_training=is_training)
else:
x = residual_block(x, name="res_block_{}".format(i),
is_training=is_training)
# A final convolution to map the features onto the right space:
with tf.variable_scope("final_pooling"):
# Batch normalization is applied first:
x = tf.layers.batch_normalization(x,
axis=-1,
momentum=0.99,
epsilon=0.001,
center=True,
scale=True,
beta_initializer=tf.zeros_initializer(),
gamma_initializer=tf.ones_initializer(),
moving_mean_initializer=tf.zeros_initializer(),
moving_variance_initializer=tf.ones_initializer(),
beta_regularizer=None,
gamma_regularizer=None,
training=is_training,
trainable=is_training,
name="BatchNorm",
reuse=None)
# ReLU:
x = tf.nn.relu(x, name="final_pooling")
x = tf.layers.conv2d(x,
n_output_classes,
kernel_size=(1, 1),
strides=(1, 1),
padding='same',
data_format='channels_last',
dilation_rate=(1, 1),
activation=None,
use_bias=False,
kernel_initializer=None, # automatically uses Xavier initializer
bias_initializer=tf.zeros_initializer(),
kernel_regularizer=None,
bias_regularizer=None,
activity_regularizer=None,
trainable=is_training,
name="Conv2DBottleNeck",
# name="convolution_globalpool_bottleneck1x1",
reuse=None)
# For global average pooling, need to get the shape of the input:
shape = (x.shape[1], x.shape[2])
x = tf.nn.pool(x,
window_shape=shape,
pooling_type="AVG",
padding="VALID",
dilation_rate=None,
strides=None,
name="GlobalAveragePool",
data_format=None)
# Reshape to remove empty dimensions:
x = tf.reshape(x, [tf.shape(x)[0], n_output_classes],
name="global_pooling_reshape")
# Apply the activation:
x = tf.nn.softmax(x, dim=-1)
return x
def _build_network(self, input_placeholder):
x = input_placeholder
# We break up the intial filters into parallel U ResNets
# The filters are concatenated at the deepest level
# And then they are split again into the parallel chains
verbose = False
# print x.get_shape()
n_planes = self._params['NPLANES']
if self._params['SHARE_PLANE_WEIGHTS']:
sharing = True
else:
sharing = False
x = tf.split(x, n_planes*[1], -1)
# for p in range(len(x)):
# print x[p].get_shape()
# Initial convolution to get to the correct number of filters:
for p in range(len(x)):
name = "Conv2DInitial"
reuse = False
if not sharing:
name += "_plane{0}".format(p)
if sharing and p != 0:
reuse = True
if verbose:
print "Name: {0} + reuse: {1}".format(name, reuse)
x[p] = tf.layers.conv2d(x[p], self._params['N_INITIAL_FILTERS'],
kernel_size=[7, 7],
strides=[1, 1],
padding='same',
use_bias=False,
trainable=self._params['TRAINING'],
name=name,
reuse=reuse)
# ReLU:
x[p] = tf.nn.relu(x[p])
# for p in range(len(x)):
# print x[p].get_shape()
# Need to keep track of the outputs of the residual blocks before downsampling, to feed
# On the upsampling side
network_filters = [[] for p in range(len(x))]
# Begin the process of residual blocks and downsampling:
for p in xrange(len(x)):
for i in xrange(self._params['NETWORK_DEPTH']):
for j in xrange(self._params['RESIDUAL_BLOCKS_PER_LAYER']):
name = "resblock_down"
reuse = False
if not sharing:
name += "_plane{0}".format(p)
if sharing and p != 0:
reuse = True
name += "_{0}_{1}".format(i, j)
if verbose:
print "Name: {0} + reuse: {1}".format(name, reuse)
x[p] = residual_block(x[p], self._params['TRAINING'],
batch_norm=self._params['BATCH_NORM'],
name=name,
reuse=reuse)
name = "downsample"
reuse = False
if not sharing:
name += "_plane{0}".format(p)
if sharing and p != 0:
reuse = True
name += "_{0}".format(i)
if verbose:
print "Name: {0} + reuse: {1}".format(name, reuse)
network_filters[p].append(x[p])
x[p] = downsample_block(x[p], self._params['TRAINING'],
batch_norm=self._params['BATCH_NORM'],
name=name,
reuse=reuse)
# print "Plane {p}, layer {i}: x[{p}].get_shape(): {s}".format(
# p=p, i=i, s=x[p].get_shape())
# print "Reached the deepest layer."
# Here, concatenate all the planes together before the residual block:
x = tf.concat(x, axis=-1)
# print "Shape after concat: " + str(x.get_shape())
# At the bottom, do another residual block:
for j in xrange(self._params['RESIDUAL_BLOCKS_DEEPEST_LAYER']):
x = residual_block(x, self._params['TRAINING'],
batch_norm=self._params['BATCH_NORM'], name="deepest_block_{0}".format(j))
# print "Shape after deepest block: " + str(x.get_shape())
# Need to split the network back into n_planes
# The deepest block doesn't change the shape, so
# it's easy to split:
x = tf.split(x, n_planes, -1)
# for p in range(len(x)):
# print x[p].get_shape()
# print "Upsampling now."
# Come back up the network:
for p in xrange(len(x)):
for i in xrange(self._params['NETWORK_DEPTH']-1, -1, -1):
# print "Up start, Plane {p}, layer {i}: x[{p}].get_shape(): {s}".format(
# p=p, i=i, s=x[p].get_shape())
# How many filters to return from upsampling?
n_filters = network_filters[p][-1].get_shape().as_list()[-1]
name = "upsample"
reuse = False
if not sharing:
name += "_plane{0}".format(p)
if sharing and p != 0:
reuse = True
name += "_{0}".format(i)
if verbose:
print "Name: {0} + reuse: {1}".format(name, reuse)
# Upsample:
x[p] = upsample_block(x[p],
self._params['TRAINING'],
batch_norm=self._params['BATCH_NORM'],
n_output_filters=n_filters,
name=name,
reuse=reuse)
x[p] = tf.concat([x[p], network_filters[p][-1]],
axis=-1, name='up_concat_plane{0}_{1}'.format(p,i))
# Remove the recently concated filters:
network_filters[p].pop()
# with tf.variable_scope("bottleneck_plane{0}_{1}".format(p,i)):
name = "BottleneckUpsample"
reuse = False
if not sharing:
name += "_plane{0}".format(p)
if sharing and p != 0:
reuse = True
name += "_{0}".format(i)
if verbose:
print "Name: {0} + reuse: {1}".format(name, reuse)
# Include a bottleneck to reduce the number of filters after upsampling:
x[p] = tf.layers.conv2d(x[p],
n_filters,
kernel_size=[1,1],
strides=[1,1],
padding='same',
activation=None,
use_bias=False,
reuse=reuse,
trainable=self._params['TRAINING'],
name=name)
x[p] = tf.nn.relu(x[p])
# Residual
for j in xrange(self._params['RESIDUAL_BLOCKS_PER_LAYER']):
name = "resblock_up"
reuse = False
if not sharing:
name += "_plane{0}".format(p)
if sharing and p != 0:
reuse = True
name += "_{0}_{1}".format(i, j)
if verbose:
print "Name: {0} + reuse: {1}".format(name, reuse)
x[p] = residual_block(x[p], self._params['TRAINING'],
batch_norm=self._params['BATCH_NORM'],
reuse=reuse,
name=name)
# print "Up end, Plane {p}, layer {i}: x[{p}].get_shape(): {s}".format(
# p=p, i=i, s=x[p].get_shape())
# Split here for segmentation labeling and vertex finding.
presplit_filters = [ layer for layer in x ]
for p in xrange(len(x)):
name = "FinalResidualBlock"
reuse = False
if not sharing:
name += "_plane{0}".format(p)
if sharing and p != 0:
reuse = True
if verbose:
print "Name: {0} + reuse: {1}".format(name, reuse)
x[p] = residual_block(x[p],
self._params['TRAINING'],
batch_norm=self._params['BATCH_NORM'],
reuse=reuse,
name=name)
name = "BottleneckConv2D"
reuse = False
if not sharing:
name += "_plane{0}".format(p)
if sharing and p != 0:
reuse = True
if verbose:
print "Name: {0} + reuse: {1}".format(name, reuse)
# At this point, we ought to have a network that has the same shape as the initial input, but with more filters.
# We can use a bottleneck to map it onto the right dimensions:
x[p] = tf.layers.conv2d(x[p],
self._params['NUM_LABELS'],
kernel_size=[7,7],
strides=[1, 1],
padding='same',
activation=None,
use_bias=False,
trainable=self._params['TRAINING'],
reuse=reuse,
name=name)
seg_logits = x
# print x[p].get_shape()
# The final activation is softmax across the pixels. It gets applied in the loss function
# x = tf.nn.softmax(x)
if self._params['VERTEX_FINDING']:
x_vtx = presplit_filters
for p in xrange(len(x_vtx)):
name = "FinalResidualBlockVertex"
reuse = False
if not sharing:
name += "_plane{0}".format(p)
if sharing and p != 0:
reuse = True
if verbose:
print "Name: {0} + reuse: {1}".format(name, reuse)
x_vtx[p] = residual_block(x_vtx[p],
self._params['TRAINING'],
batch_norm=self._params['BATCH_NORM'],
reuse=reuse,
name=name)
name = "BottleneckConv2DVertex"
reuse = False
if not sharing:
name += "_plane{0}".format(p)
if sharing and p != 0:
reuse = True
if verbose:
print "Name: {0} + reuse: {1}".format(name, reuse)
# At this point, we ought to have a network that has the same shape as the initial input, but with more filters.
# We can use a bottleneck to map it onto the right dimensions:
x_vtx[p] = tf.layers.conv2d(x_vtx[p],
1,
kernel_size=[5,5],
strides=[1, 1],
padding='same',
activation=None,
use_bias=False,
trainable=self._params['TRAINING'],
reuse=reuse,
name=name)
# This comes out with one filter but it should really by one dimension reduced:
x_vtx[p] = tf.squeeze(x_vtx[p], axis=-1)
vertex_logits = x_vtx
else:
vertex_logits = None
return seg_logits, vertex_logits
def get_training_model(width, height, bs=1, bi_style=False):
input_o = layers.Input(shape=(height, width, 3),
dtype='float32',
name='input_o')
c1 = layers.Conv2D(32, (9, 9), strides=1, padding='same',
name='conv_1')(input_o)
c1 = layers.BatchNormalization(name='normal_1')(c1)
c1 = layers.Activation('relu', name='relu_1')(c1)
c2 = layers.Conv2D(64, (3, 3), strides=2, padding='same',
name='conv_2')(c1)
c2 = layers.BatchNormalization(name='normal_2')(c2)
c2 = layers.Activation('relu', name='relu_2')(c2)
c3 = layers.Conv2D(128, (3, 3), strides=2, padding='same',
name='conv_3')(c2)
c3 = layers.BatchNormalization(name='normal_3')(c3)
c3 = layers.Activation('relu', name='relu_3')(c3)
r1 = residual_block(c3, 1)
r2 = residual_block(r1, 2)
r3 = residual_block(r2, 3)
r4 = residual_block(r3, 4)
r5 = residual_block(r4, 5)
d1 = layers.Conv2DTranspose(64, (3, 3),
strides=2,
padding='same',
name='conv_4')(r5)
d1 = layers.BatchNormalization(name='normal_4')(d1)
d1 = layers.Activation('relu', name='relu_4')(d1)
d2 = layers.Conv2DTranspose(32, (3, 3),
strides=2,
padding='same',
name='conv_5')(d1)
d2 = layers.BatchNormalization(name='normal_5')(d2)
d2 = layers.Activation('relu', name='relu_5')(d2)
c4 = layers.Conv2D(3, (9, 9), strides=1, padding='same', name='conv_6')(d2)
c4 = layers.BatchNormalization(name='normal_6')(c4)
c4 = layers.Activation('tanh', name='tanh_1')(c4)
c4 = OutputScale(name='output')(c4)
content_activation = layers.Input(shape=(height // 2, width // 2, 128),
dtype='float32')
style_activation1 = layers.Input(shape=(height, width, 64),
dtype='float32')
style_activation2 = layers.Input(shape=(height // 2, width // 2, 128),
dtype='float32')
style_activation3 = layers.Input(shape=(height // 4, width // 4, 256),
dtype='float32')
style_activation4 = layers.Input(shape=(height // 8, width // 8, 512),
dtype='float32')
if bi_style:
style_activation1_2 = layers.Input(shape=(height, width, 64),
dtype='float32')
style_activation2_2 = layers.Input(shape=(height // 2, width // 2,
128),
dtype='float32')
style_activation3_2 = layers.Input(shape=(height // 4, width // 4,
256),
dtype='float32')
style_activation4_2 = layers.Input(shape=(height // 8, width // 8,
512),
dtype='float32')
total_variation_loss = layers.Lambda(get_tv_loss,
output_shape=(1, ),
name='tv',
arguments={
'width': width,
'height': height
})([c4])
# Block 1
x = layers.Conv2D(64, (3, 3),
activation='relu',
padding='same',
name='block1_conv1')(c4)
x = layers.Conv2D(64, (3, 3),
activation='relu',
padding='same',
name='block1_conv2')(x)
style_loss1 = layers.Lambda(get_style_loss,
output_shape=(1, ),
name='style1',
arguments={'batch_size':
bs})([x, style_activation1])
if bi_style:
style_loss1_2 = layers.Lambda(get_style_loss,
output_shape=(1, ),
name='style1_2',
arguments={'batch_size':
bs})([x, style_activation1_2])
style_loss1 = AverageAddTwo(name='style1_out')(
[style_loss1, style_loss1_2])
x = layers.MaxPooling2D((2, 2), strides=(2, 2), name='block1_pool')(x)
# Block 2
x = layers.Conv2D(128, (3, 3),
activation='relu',
padding='same',
name='block2_conv1')(x)
x = layers.Conv2D(128, (3, 3),
activation='relu',
padding='same',
name='block2_conv2')(x)
content_loss = layers.Lambda(get_content_loss,
output_shape=(1, ),
name='content')([x, content_activation])
style_loss2 = layers.Lambda(get_style_loss,
output_shape=(1, ),
name='style2',
arguments={'batch_size':
bs})([x, style_activation2])
if bi_style:
style_loss2_2 = layers.Lambda(get_style_loss,
output_shape=(1, ),
name='style2_2',
arguments={'batch_size':
bs})([x, style_activation2_2])
style_loss2 = AverageAddTwo(name='style2_out')(
[style_loss2, style_loss2_2])
x = layers.MaxPooling2D((2, 2), strides=(2, 2), name='block2_pool')(x)
# Block 3
x = layers.Conv2D(256, (3, 3),
activation='relu',
padding='same',
name='block3_conv1')(x)
x = layers.Conv2D(256, (3, 3),
activation='relu',
padding='same',
name='block3_conv2')(x)
x = layers.Conv2D(256, (3, 3),
activation='relu',
padding='same',
name='block3_conv3')(x)
style_loss3 = layers.Lambda(get_style_loss,
output_shape=(1, ),
name='style3',
arguments={'batch_size':
bs})([x, style_activation3])
if bi_style:
style_loss3_2 = layers.Lambda(get_style_loss,
output_shape=(1, ),
name='style3_2',
arguments={'batch_size':
bs})([x, style_activation3_2])
style_loss3 = AverageAddTwo(name='style3_out')(
[style_loss3, style_loss3_2])
x = layers.MaxPooling2D((2, 2), strides=(2, 2), name='block3_pool')(x)
# Block 4
x = layers.Conv2D(512, (3, 3),
activation='relu',
padding='same',
name='block4_conv1')(x)
x = layers.Conv2D(512, (3, 3),
activation='relu',
padding='same',
name='block4_conv2')(x)
x = layers.Conv2D(512, (3, 3),
activation='relu',
padding='same',
name='block4_conv3')(x)
style_loss4 = layers.Lambda(get_style_loss,
output_shape=(1, ),
name='style4',
arguments={'batch_size':
bs})([x, style_activation4])
if bi_style:
style_loss4_2 = layers.Lambda(get_style_loss,
output_shape=(1, ),
name='style4_2',
arguments={'batch_size':
bs})([x, style_activation4_2])
style_loss4 = AverageAddTwo(name='style4_out')(
[style_loss4, style_loss4_2])
x = layers.MaxPooling2D((2, 2), strides=(2, 2), name='block4_pool')(x)
# Block 5
x = layers.Conv2D(512, (3, 3),
activation='relu',
padding='same',
name='block5_conv1')(x)
x = layers.Conv2D(512, (3, 3),
activation='relu',
padding='same',
name='block5_conv2')(x)
x = layers.Conv2D(512, (3, 3),
activation='relu',
padding='same',
name='block5_conv3')(x)
x = layers.MaxPooling2D((2, 2), strides=(2, 2), name='block5_pool')(x)
if bi_style:
model = Model([
input_o, content_activation, style_activation1, style_activation2,
style_activation3, style_activation4, style_activation1_2,
style_activation2_2, style_activation3_2, style_activation4_2
], [
content_loss, style_loss1, style_loss2, style_loss3, style_loss4,
total_variation_loss, c4
])
else:
model = Model([
input_o, content_activation, style_activation1, style_activation2,
style_activation3, style_activation4
], [
content_loss, style_loss1, style_loss2, style_loss3, style_loss4,
total_variation_loss, c4
])
model_layers = {layer.name: layer for layer in model.layers}
original_vgg = vgg16.VGG16(weights='imagenet', include_top=False)
original_vgg_layers = {layer.name: layer for layer in original_vgg.layers}
# load image_net weight
for layer in original_vgg.layers:
if layer.name in model_layers:
model_layers[layer.name].set_weights(
original_vgg_layers[layer.name].get_weights())
model_layers[layer.name].trainable = False
print("training model built successfully!")
return model
def __init__(self,
sess,
ob_space,
ac_space,
n_env,
n_steps,
n_batch,
n_lstm=256,
reuse=False,
layers=None,
cnn_extractor=nature_cnn,
layer_norm=False,
feature_extraction="cnn",
**kwargs):
# super(LstmPolicy, self).__init__(sess, ob_space, ac_space, n_env, n_steps, n_batch, n_lstm, reuse,
# scale=(feature_extraction == "cnn"))
# add this function to LstmPolicy to init ActorCriticPolicy
self.AC_init(sess, ob_space, ac_space, n_env, n_steps, n_batch, n_lstm,
reuse, feature_extraction)
with tf.variable_scope("model", reuse=reuse):
extracted_features = cnn_extractor(self.processed_x,
**kwargs) # # [B,H,W,Deepth]
print('extracted_features', extracted_features)
coor = get_coor(extracted_features)
# [B,Height,W,D+2]
entities = tf.concat([extracted_features, coor], axis=3)
print('entities:', entities)
# [B,H*W,num_heads,Deepth=D+2]
MHDPA_output, weights = MHDPA(entities,
"extracted_features",
num_heads=2)
print('MHDPA_output', MHDPA_output)
self.attention = weights
# [B,H*W,num_heads,Deepth]
residual_output = residual_block(entities, MHDPA_output)
print('residual_output', residual_output)
# max_pooling
residual_maxpooling_output = tf.reduce_max(residual_output,
axis=[1])
print('residual_maxpooling_output', residual_maxpooling_output)
input_sequence = batch_to_seq(residual_maxpooling_output,
self.n_env, n_steps)
# input_sequence = batch_to_seq(extracted_features, self.n_env, n_steps)
masks = batch_to_seq(self.masks_ph, self.n_env, n_steps)
rnn_output, self.snew = lstm(input_sequence,
masks,
self.states_ph,
'lstm1',
n_hidden=n_lstm,
layer_norm=layer_norm)
rnn_output = seq_to_batch(rnn_output)
# print('rnn_output', rnn_output, ' snew', self.snew)
value_fn = linear(rnn_output, 'vf', 1)
self.proba_distribution, self.policy, self.q_value = \
self.pdtype.proba_distribution_from_latent(rnn_output, rnn_output)
self.value_fn = value_fn
self.initial_state = np.zeros((self.n_env, n_lstm * 2),
dtype=np.float32)
self._setup_init()
def build_discriminator(input_tensor,
n_initial_filters=12,
n_blocks = 2,
is_training=True,
alpha=0.2,
reuse=False):
"""
Build a DC GAN discriminator with deep convolutional layers
Input_tensor is assumed to be reshaped into a (BATCH, L, H, F) type format (rectangular)
"""
with tf.variable_scope("discriminator", reuse=reuse):
# Map the input to a small but many-filtered set up:
x = tf.layers.conv2d(input_tensor,
n_initial_filters,
kernel_size=[3, 3],
strides=[1, 1],
padding='same',
activation=None,
use_bias=False,
kernel_initializer=None, # automatically uses Xavier initializer
kernel_regularizer=None,
activity_regularizer=None,
trainable=is_training,
name="Conv2D",
reuse=reuse)
# Apply residual mappings and upsamplings:
for block in xrange(n_blocks):
x = residual_block(x, is_training = is_training,
kernel=[3, 3], stride=[1, 1],
alpha=alpha,
name="res_block_{}".format(block),
reuse=reuse)
x = downsample_block(x, is_training = is_training,
kernel=[3, 3],
stride=[1, 1],
alpha=alpha,
name="res_block_upsample_{}".format(block),
reuse=reuse)
# Apply a 1x1 convolution to map to just one output filter:
x = tf.layers.conv2d(x,
1,
kernel_size=[3, 3],
strides=[1, 1],
padding='same',
activation=None,
use_bias=False,
kernel_initializer=None, # automatically uses Xavier initializer
kernel_regularizer=None,
activity_regularizer=None,
trainable=is_training,
name="FinalConv2D1x1",
reuse=reuse)
#Apply global average pooling to the final layer, then sigmoid activation
# For global average pooling, need to get the shape of the input:
shape = (x.shape[1], x.shape[2])
x = tf.nn.pool(x,
window_shape=shape,
pooling_type="AVG",
padding="VALID",
dilation_rate=None,
strides=None,
name="GlobalAveragePool",
data_format=None)
x = tf.reshape(x, (-1, 1))
# For the final activation, apply sigmoid:
return tf.nn.sigmoid(x)
def build_discriminator_progressive(input_tensor,
leaky_relu_param=0.0,
n_filters=64,
n_blocks = 2,
is_training=True,
reuse=True,):
"""
This function will build a discriminator to decide if an image is real
or fake.
Starting at a low resolution of 4x4 output, and gradually increasing
To make it easy to reuse weights, every level has a fixed number of
filters
"""
with tf.variable_scope("discriminator_progressive", reuse = tf.AUTO_REUSE):
# Map the initial set of random numbers to a 4x4xn_initial_filters space:
x = tf.layers.conv2d(input_tensor,
n_filters,
kernel_size=[3, 3],
strides=[1, 1],
padding='same',
activation=None,
use_bias=False,
kernel_initializer=None, # automatically uses Xavier initializer
kernel_regularizer=None,
activity_regularizer=None,
trainable=is_training,
name="Conv2D")
current_size = int(x.get_shape()[1])
while current_size > 4:
current_size = int(x.get_shape()[1])
next_size = int(0.5*current_size)
subname = "{}to{}".format(current_size, next_size)
x = residual_block(x,
is_training,
alpha=leaky_relu_param,
name="res_block_{}".format(subname))
x = downsample_block(x,
is_training,
alpha=leaky_relu_param,
name="downsample_block_{}".format(subname))
# For global average pooling, need to get the shape of the input:
shape = (x.shape[1], x.shape[2])
x = tf.nn.pool(x,
window_shape=shape,
pooling_type="AVG",
padding="VALID",
dilation_rate=None,
strides=None,
name="GlobalAveragePool",
data_format=None)
x = tf.reshape(x, (-1, 1))
# For the final activation, apply sigmoid:
return tf.nn.sigmoid(x)
return x