mu = Dense(n_z, activation='linear')(h_q)
log_sigma = Dense(n_z, activation='linear')(h_q)
# In[7]:
def sample_z(args):
mu, log_sigma = args
eps = K.random_normal(shape=(m, n_z), mean=0., stddev=epsilon_std)
return mu + K.exp(log_sigma / 2) * eps
# In[8]:
# Sample z ~ Q(z|X)
z = Lambda(sample_z)([mu, log_sigma])
# In[9]:
# P(X|z) -- decoder
decoder_hidden = Dense(512, activation='relu')
decoder_out = Dense(784, activation='sigmoid')
h_p = decoder_hidden(z)
outputs = decoder_out(h_p)
# In[10]:
# Overall VAE model, for reconstruction and training
vae = Model(inputs, outputs)
import tensorflow as tf
from tensorflow.keras.layers import Lambda
def do_from_complex(x):
real = tf.math.real(x)
imag = tf.math.imag(x)
zed = tf.stack([real, imag], axis=-1)
return zed
def do_to_complex(x):
real = x[..., 0]
imag = x[..., 1]
result = tf.complex(real, imag)
return result
from_complex = Lambda(lambda x: do_from_complex(x), name='from_complex')
to_complex = Lambda(lambda x: do_to_complex(x), name='to_complex')
def large_model(input_shape, label_num, max_text_len, train=True):
"""
input_shape: as batching is used, should be of shape (None, j, k) where j = number of time steps
and k = number of features per time step
label_num: a dictionary of characters to be used in calculating CTC loss
max_text_length: the maximum length of the ground truth sentences
train: boolean for training or not, defines the input and output structure of the model
"""
X_input = Input(batch_shape=input_shape, name='input')
# First conv block
X = Conv1D(256,
11,
strides=1,
padding='same',
dilation_rate=1,
kernel_initializer='he_normal',
kernel_regularizer=regularizers.l2(0.01),
name='conv_input')(X_input)
X = BatchNormalization(momentum=0.997)(X)
# Use relu activation function
X = ReLU()(X)
X = Dropout(0.20)(X)
# Add residual blocks
X = res_block(X, 256, 11, 0.2, 1, 5, 0)
X = res_block(X, 256, 11, 0.2, 1, 5, 1)
X = res_block(X, 384, 13, 0.2, 2, 5, 0)
X = res_block(X, 384, 13, 0.2, 2, 5, 1)
X = res_block(X, 512, 17, 0.2, 3, 5, 0)
X = res_block(X, 512, 17, 0.2, 3, 5, 1)
X = res_block(X, 640, 21, 0.3, 4, 5, 0)
X = res_block(X, 640, 21, 0.3, 4, 5, 1)
X = res_block(X, 768, 25, 0.3, 5, 5, 0)
X = res_block(X, 768, 25, 0.3, 5, 5, 1)
# Final conv layers
X = Conv1D(896,
29,
strides=1,
padding='same',
dilation_rate=2,
kernel_initializer='he_normal',
kernel_regularizer=regularizers.l2(0.01),
name='conv_dil2')(X)
X = BatchNormalization(momentum=0.997)(X)
X = ReLU()(X)
X = Dropout(0.40)(X)
X = Conv1D(1024,
1,
strides=1,
padding='same',
dilation_rate=1,
kernel_initializer='he_normal',
kernel_regularizer=regularizers.l2(0.01),
name='conv_end1')(X)
X = BatchNormalization(momentum=0.997)(X)
X = ReLU()(X)
X = Dropout(0.40)(X)
# Final fc layer with softmax output
X = Conv1D(label_num,
1,
strides=1,
padding='same',
dilation_rate=1,
kernel_initializer='he_normal',
kernel_regularizer=regularizers.l2(0.01),
name='conv_end2')(X)
y_pred = Softmax(name='softmax')(X)
if train:
# Section regarding the implementation of CTC loss function
labels = Input(name='the_labels',
shape=(max_text_len, ),
dtype='int16')
input_length = Input(name='input_length', shape=(1, ), dtype='int16')
label_length = Input(name='label_length', shape=(1, ), dtype='int16')
# Keras doesn't currently support loss funcs with extra parameters
# so CTC loss is implemented in a lambda layer
loss_out = Lambda(ctc_lambda_func, output_shape=(1, ), name='ctc')(
[labels, y_pred, input_length, label_length]) #(None, 1)
model = Model(inputs=[X_input, labels, input_length, label_length],
outputs=loss_out)
# Clipnorm seems to speeds up convergence
sgd = SGD(lr=0.001,
decay=1e-6,
momentum=0.9,
nesterov=True,
clipnorm=5)
# The loss calc occurs elsewhere, so use a dummy lambda func for the loss
model.compile(loss={
'ctc': lambda y_true, y_pred: y_pred
},
optimizer=sgd)
else:
model = Model(inputs=X_input, outputs=y_pred)
return model
def __init__(self,
corpus,
batch_size,
filters,
kernel_size,
hidden_units,
dropout_rate=0,
d1=0,
d2=0,
trainable_embed=False):
"""
Args:
- corpus: the AMI_Corpus object (see utils)
- batch_size: batch_size for training
- filters: number of convolutional filters
- kernel_size: kernel size of filters
- hidden_units: size of feed-forward layers
- dropout_rate: rate for dropout layers
- d1, d2: history parameters
- trainable_embed: whether we want our embedding layer (which has pretrained word vectors) to be updated
during training
"""
self.corpus = corpus
seq_length = d1 + d2 + 1
embed_dim = corpus.embed_dim
# input shape is [batch_size, seq_length, utterance_length]
inputs_ = Input(shape=(seq_length, self.corpus.max_utt_length),
name="input")
# embedding
x_ = embedding_layer(corpus=self.corpus,
embed_dim=embed_dim,
inputs=inputs_,
trainable=trainable_embed)
# output: [batch_size, seq_length, utt_length, embed_dim]
# convolutional layers
stv_s_ = []
for i in range(seq_length):
# selecting utterance at a time
u_ = Lambda(lambda x: x[:, i, :, :], name="Lambda_" + str(i))(x_)
# shape is now [batch_size, utt_length, embed_dim]
u_ = Conv1D(filters=filters,
kernel_size=kernel_size,
activation='relu',
name='Conv_' + str(i))(u_)
# shape is now [batch_size, (utt_length - kernel_size + 1), filters]
stv_ = GlobalMaxPooling1D(name='glob_max_pool_' + str(i))(u_)
# shape is now [batch_size, filters]
# Lee and Dernoncourt applied dropout here
stv_ = Dropout(rate=dropout_rate, name="dropout_" + str(i))(stv_)
stv_s_.append(stv_)
# we now construct FF layers for t-d2 ..., t-1, t
ff_outputs = []
for f in range(-d2 - 1, 0):
# every FF layer f takes as inputs f-d1, ... f-1, f
if d1 == 0: # d1 is 0; every FF unit takes one stv as input
s_ = stv_s_[f]
elif d1 != 0 and f == -1:
# last FF network ... take right d1+1 inputs
s_ = stv_s_[f - d1:]
s_ = Concatenate(axis=-1)(s_)
else:
s_ = stv_s_[f - d1:f + 1]
s_ = Concatenate(axis=-1)(s_)
# shape is now [batch, filters * (d1+1)]
ff_ = Dense(hidden_units, activation='relu',
name='FF_' + str(f))(s_)
# shape is now [batch_size, hidden_units]
ff_outputs.append(ff_)
# concatenate FF outputs (if necessary)
if len(ff_outputs) == 1:
ff_t_ = ff_outputs[0]
else:
ff_t_ = Concatenate(axis=-1)(ff_outputs)
# shape is now [batch_size, (d2+1) * hidden_units]
# output layer
predictions_ = Dense(17, activation='softmax',
name='softmax_output')(ff_t_)
self.model = Model(inputs=inputs_, outputs=predictions_)
def call(self, inp):
images, vertexlabel, Js_in = inp
out_dict = {}
images = [tf.Variable(x, dtype=tf.float32, trainable=False) for x in images]
vertexlabel = tf.cast(tf.Variable(vertexlabel, trainable=False), tf.int32)
if FACE:
Js = [Lambda(lambda j: j[:, :25])(J) for J in Js_in]
else:
Js = [self.flatten(tf.cast(tf.Variable(x, trainable=False), tf.float32)) for x in Js_in]
with tf.device('/gpu:0'):
lat_codes = [self.top_([q, j]) for q, j in zip(images, Js)]
latent_code_offset = self.avg([q[0] for q in lat_codes]) if (len(lat_codes) > 1 ) else lat_codes[0][0]
latent_code_betas = self.avg([q[1] for q in lat_codes]) if (len(lat_codes) > 1 ) else lat_codes[0][1]
latent_code_pose = [tf.concat([q[1], x], axis=-1) for q, x in zip(lat_codes, Js)]
with tf.device('/gpu:0'):
latent_code_betas = self.lat_betas(latent_code_betas)
betas = self.betas(latent_code_betas)
latent_code_pose = [self.lat_pose(x) for x in latent_code_pose]
pose_trans_init = tf.tile(tf.expand_dims(self.pose_trans, 0), (K.int_shape(betas)[0], 1))
poses_ = [self.lat_pose_layer(x) + pose_trans_init for x in latent_code_pose]
trans_ = [self.cut_trans(x) for x in poses_]
trans = [la(i) for la, i in zip(self.trans_layers, trans_)]
poses_ = [self.cut_poses(x) for x in poses_]
poses_ = [self.reshape_pose(x) for x in poses_]
poses = [la(i) for la, i in zip(self.pose_layers, poses_)]
##
out_dict['betas'] = betas
for i in range(NUM):
out_dict['pose_{}'.format(i)] = poses[i]
out_dict['trans_{}'.format(i)] = trans[i]
latent_code_offset_ShapeMerged = self.latent_code_offset_ShapeMerged(latent_code_offset)
latent_code_offset_ShapeMerged = self.latent_code_offset_ShapeMerged_2(latent_code_offset_ShapeMerged)
garm_model_outputs = [fe(latent_code_offset_ShapeMerged) for fe in self.garmentModels]
garment_verts_all = [fe[0] for fe in garm_model_outputs]
garment_pca = [fe[1] for fe in garm_model_outputs]
garment_pca = tf.stack(garment_pca, axis=1)
##
out_dict['pca_verts'] = garment_pca
lis = []
for go, vs in zip(garment_verts_all, self.scatters):
lis.append(vs(go))
garment_verts_all_scattered = tf.stack(lis, axis=-1)
## Get naked smpl to compute garment offsets
zerooooooo = K.zeros_like(garment_verts_all_scattered[..., 0])
pooooooooo = [K.zeros_like(p) for p in poses]
tooooooooo = [K.zeros_like(p) for p in trans]
smpls_base = []
for i, (p, t) in enumerate(zip(pooooooooo, tooooooooo)):
v, _, n, _ = self.smpl(p, betas, t, zerooooooo)
smpls_base.append(v)
if i == 0:
vertices_naked_ = n
## Append Skin offsets
garment_verts_all_scattered = tf.concat(
[K.expand_dims(vertices_naked_, -1), tf.cast(garment_verts_all_scattered, vertices_naked_.dtype)],
axis=-1)
garment_verts_all_scattered = tf.transpose(garment_verts_all_scattered, perm=[0, 1, 3, 2])
clothed_verts = tf.compat.v1.batch_gather(garment_verts_all_scattered, vertexlabel)
clothed_verts = tf.squeeze(tf.transpose(clothed_verts, perm=[0, 1, 3, 2]))
offsets_ = clothed_verts - vertices_naked_
smpls = []
for i, (p, t) in enumerate(zip(poses, trans)):
v, t, n, _ = self.smpl(p, betas, t, offsets_)
smpls.append(v)
if i == 0:
vertices_naked = n
vertices_tposed = t
Js = [jl(self.smpl_J([p, betas, t])) for jl, p, t in zip(self.J_layers, poses, trans)]
vertices = tf.concat([tf.expand_dims(smpl, axis=-1) for i, smpl in enumerate(smpls)], axis=-1)
##
out_dict['vertices'] = vertices
out_dict['vertices_tposed'] = vertices_tposed
out_dict['vertices_naked'] = vertices_naked
##
out_dict['vertices'] = vertices
out_dict['vertices_tposed'] = vertices_tposed
out_dict['vertices_naked'] = vertices_naked
out_dict['offsets_h'] = offsets_
for i in range(NUM):
out_dict['J_{}'.format(i)] = Js[i]
vert_cols = tf.reshape(tf.gather(self.colormap, tf.reshape(vertexlabel, (-1,))), (-1, config.NVERTS, 3))
renderered_garms_all = []
for view in range(NUM):
renderered_garms_all.append(render_colored_batch(vertices[..., view], self.faces, vert_cols, # [bat],
IMG_SIZE, IMG_SIZE, FOCAL_LENGTH,
CAMERA_CENTER,
np.zeros(3, dtype=np.float32),
num_channels=3))
renderered_garms_all = tf.transpose(renderered_garms_all, [1, 2, 3, 4, 0])
out_dict['rendered'] = renderered_garms_all
lap = compute_laplacian_diff(vertices_tposed, vertices_naked, self.faces)
##
out_dict['laplacian'] = lap
return out_dict
def on_epoch_end(self, epoch, logs={}):
current = logs.get(self.monitor)
if current < self.value:
print("Epoch %05d: reached desired error at epoch" % epoch)
self.model.stop_training = True
def custom_loss(y_true, y_pred):
#A = tensorflow.keras.losses.mean_squared_error(y_true, y_pred)
B = tensorflow.keras.losses.mean_absolute_error(y_true, y_pred)
return (B)
sum_dim_channel = Lambda(lambda xin: K.sum(xin, axis=3))
def lrelu(x): #from pix2pix code
a = 0.2
# adding these together creates the leak part and linear part
# then cancels them out by subtracting/adding an absolute value term
# leak: a*x/2 - a*abs(x)/2
# linear: x/2 + abs(x)/2
# this block looks like it has 2 inputs on the graph unless we do this
x = tf.identity(x)
return (0.5 * (1 + a)) * x + (0.5 * (1 - a)) * tf.abs(x)
def lrelu_output_shape(input_shape):
def __init__(self, units, num_patches, **kwargs):
super(PatchSampleMLP, self).__init__(**kwargs)
self.units = units
self.num_patches = num_patches
self.l2_norm = Lambda(lambda x: x * tf.math.rsqrt(tf.reduce_sum(tf.square(x), axis=-1, keepdims=True) + 10-10))
def nn_model(self):
obs_input = Input(self.state_dim, name="im_obs")
conv1 = Conv2D(
filters=64,
kernel_size=(3, 3),
strides=(1, 1),
padding="same",
input_shape=self.state_dim,
data_format="channels_last",
activation="relu",
)(obs_input)
pool1 = MaxPool2D(pool_size=(3, 3), strides=1)(conv1)
conv2 = Conv2D(
filters=32,
kernel_size=(3, 3),
strides=(1, 1),
padding="valid",
activation="relu",
)(pool1)
pool2 = MaxPool2D(pool_size=(3, 3), strides=1)(conv2)
conv3 = Conv2D(
filters=16,
kernel_size=(3, 3),
strides=(1, 1),
padding="valid",
activation="relu",
)(pool2)
pool3 = MaxPool2D(pool_size=(3, 3), strides=1)(conv3)
conv4 = Conv2D(
filters=8,
kernel_size=(3, 3),
strides=(1, 1),
padding="valid",
activation="relu",
)(pool3)
pool4 = MaxPool2D(pool_size=(3, 3), strides=1)(conv4)
flat = Flatten()(pool4)
dense1 = Dense(16,
activation="relu",
kernel_initializer=self.weight_initializer)(flat)
dropout1 = Dropout(0.3)(dense1)
dense2 = Dense(8,
activation="relu",
kernel_initializer=self.weight_initializer)(dropout1)
dropout2 = Dropout(0.3)(dense2)
# action_dim[0] = 2
output_val = Dense(
self.action_dim[0],
activation="relu",
kernel_initializer=self.weight_initializer,
)(dropout2)
# Scale & clip x[i] to be in range [0, action_bound[i]]
action_bound = copy.deepcopy(self.action_bound)
mu_output = Lambda(
lambda x: tf.clip_by_value(x * action_bound, 1e-9, action_bound),
name="mu_output",
)(output_val)
std_output_1 = Dense(
self.action_dim[0],
activation="softplus",
kernel_initializer=self.weight_initializer,
)(dropout2)
std_output = Lambda(lambda x: tf.clip_by_value(
x * action_bound, 1e-9, action_bound / 2, name="std_output"))(
std_output_1)
return tf.keras.models.Model(inputs=obs_input,
outputs=[mu_output, std_output],
name="Actor")
def build(self, input_shape):
with K.name_scope(
self.name
): # name scope used to make sure weights get unique names
self.layers = []
self.res_output_shape = input_shape
for k in range(2):
name = 'conv1D_{}'.format(k)
with K.name_scope(
name
): # name scope used to make sure weights get unique names
self._add_and_activate_layer(
Conv1D(filters=self.nb_filters,
kernel_size=self.kernel_size,
dilation_rate=self.dilation_rate,
padding=self.padding,
name=name,
kernel_initializer=self.kernel_initializer,
kernel_regularizer=regularizers.l2(self.l2_reg),
bias_regularizer=regularizers.l2(self.l2_reg)))
with K.name_scope('norm_{}'.format(k)):
if self.use_batch_norm:
self._add_and_activate_layer(BatchNormalization())
elif self.use_layer_norm:
self._add_and_activate_layer(LayerNormalization())
self._add_and_activate_layer(Activation('relu'))
self._add_and_activate_layer(
SpatialDropout1D(rate=self.dropout_rate))
if self.nb_filters != input_shape[-1]:
# 1x1 conv to match the shapes (channel dimension).
name = 'matching_conv1D'
with K.name_scope(name):
# make and build this layer separately because it directly uses input_shape
self.shape_match_conv = Conv1D(
filters=self.nb_filters,
kernel_size=1,
padding='same',
name=name,
kernel_initializer=self.kernel_initializer)
else:
name = 'matching_identity'
self.shape_match_conv = Lambda(lambda x: x, name=name)
with K.name_scope(name):
self.shape_match_conv.build(input_shape)
self.res_output_shape = self.shape_match_conv.compute_output_shape(
input_shape)
self.final_activation = Activation(self.activation)
self.final_activation.build(
self.res_output_shape) # probably isn't necessary
# this is done to force Keras to add the layers in the list to self._layers
for layer in self.layers:
self.__setattr__(layer.name, layer)
self.__setattr__(self.shape_match_conv.name, self.shape_match_conv)
self.__setattr__(self.final_activation.name, self.final_activation)
super(ResidualBlock, self).build(
input_shape) # done to make sure self.built is set True
from tensorflow.python.ops import math_ops
from kernels import sobel_x
from artist import CustomImage, ImageBundle, InputImage, OutputImage
import pickle
import sys
from tensorflow.keras.layers import Input, MaxPool2D, Conv2D, BatchNormalization, Dropout, Flatten, Concatenate, Dense, Reshape, Activation, Lambda, LeakyReLU
# Define Sobel filter
sobel_x = tf.constant_initializer(sobel_x())
# TensorFlow expects 4D tensors of shape (samples, rows, cols, channels)
# Note that the first index (the sample index out of the batch) is stripped
model_input = Input(shape=(512, 512, 1))
Lambda_In = Lambda(lambda x: x / 255.)(model_input)
#Pool1 = MaxPool2D(pool_size=(2, 2))(model_input) # (256, 256, 1)
Pool2 = MaxPool2D(pool_size=(8, 8))(Lambda_In) # (164, 64, 1)
#Pool3 = MaxPool2D(pool_size=(2, 2))(Pool2) # (64, 64, 1)
#Pool4 = MaxPool2D(pool_size=(2, 2))(Pool3) # (32, 32, 1)
#Pool5 = MaxPool2D(pool_size=(2, 2))(Pool4) # (16, 16, 1)
Conv21 = Conv2D(128, (3, 3),
padding='same',
kernel_initializer=sobel_x,
data_format='channels_last')(Pool2) # (128, 128, 30)
Activ21 = Activation('sigmoid')(Conv21)
BN21 = BatchNormalization(axis=2)(Activ21)
Drop21 = Dropout(0.1)(BN21)
Conv22 = Conv2D(128, (5, 5), padding='same',
def build_encoder(self):
initializer = RandomNormal(mean=0.0, stddev=0.05, seed=None)
self.inputs = Input(shape=self.img_shape, name='Encoder_input')
conv1_1 = Conv2D(32,
kernel_size=(3, 3),
kernel_initializer=initializer,
strides=(1, 1),
padding='same',
activation='relu')(self.inputs)
conv1_2 = Conv2D(32,
kernel_size=(3, 3),
kernel_initializer=initializer,
strides=(1, 1),
padding='same',
activation='relu')(conv1_1)
pool1 = MaxPooling2D(padding="same")(conv1_2)
conv1_3 = Conv2D(64,
kernel_size=(3, 3),
kernel_initializer=initializer,
strides=(1, 1),
padding='same',
activation='relu')(pool1)
conv1_4 = Conv2D(64,
kernel_size=(3, 3),
kernel_initializer=initializer,
strides=(1, 1),
padding='same',
activation='relu')(conv1_3)
conv1_5 = Conv2D(64,
kernel_size=(3, 3),
kernel_initializer=initializer,
strides=(1, 1),
padding='same',
activation='relu')(conv1_4)
pool2 = MaxPooling2D(padding="same")(conv1_5)
conv1_6 = Conv2D(64,
kernel_size=(3, 3),
kernel_initializer=initializer,
strides=(1, 1),
padding='same',
activation='relu')(pool2)
conv1_7 = Conv2D(64,
kernel_size=(3, 3),
kernel_initializer=initializer,
strides=(1, 1),
padding='same',
activation='relu')(conv1_6)
conv_flat = Flatten()(conv1_7)
fc_1 = Dense(64, activation='relu',
kernel_initializer=initializer)(conv_flat)
self.z_mean = Dense(self.latent_dim, name='z_mean')(fc_1)
self.z_log_var = Dense(self.latent_dim, name='z_log_var')(fc_1)
z = Lambda(self.sampling, output_shape=(self.latent_dim, ),
name='z')([self.z_mean, self.z_log_var])
return Model(self.inputs, [self.z_mean, self.z_log_var, z],
name='encoder')
x = Conv2D(256, (3, 3), strides=(1, 1), padding="same")(x)
x = MaxPooling2D(pool_size=(2, 2), strides=(2, 2))(x)
x = Conv2D(512, (3, 3), strides=(1, 1), padding="same")(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2D(512, (3, 3), strides=(1, 1), padding="same")(x)
x = MaxPooling2D(pool_size=(1, 2), strides=(1, 2))(x)
x = ZeroPadding2D(padding=(1, 0), data_format='channels_last')(x)
x = Conv2D(512, (3, 3), strides=(1, 1))(x)
top = Lambda(top_norm)(x)
top = Reshape(target_shape=(12, 1, 512))(top)
bottom = Lambda(bottom_norm)(x)
bottom = Reshape(target_shape=(12, 1, 512))(bottom)
x = concatenate([top, bottom], 1)
x = Reshape(target_shape=(24, 512))(x)
gru_1 = GRU(128,
return_sequences=True,
kernel_initializer='he_normal',
name='gru1')(x)
gru_1b = GRU(128,
return_sequences=True,
go_backwards=True,
kernel_initializer='he_normal',
def __init__(self,
n_z,
stddev_epsilon=1e-6,
final_activation='sigmoid',
trainable_encoder=True,
trainable_decoder=[True] * 5,
res=28):
super(VAEModel, self).__init__()
self.n_z = n_z
self.stddev_epsilon = stddev_epsilon
self.res = res
self.final_activation = final_activation
self.trainable_decoder = trainable_decoder
# Encoder architecture
self.conv1 = Conv2D(filters=64,
kernel_size=4,
strides=2,
trainable=trainable_encoder)
self.bne1 = BatchNormalization(momentum=0.9,
epsilon=1e-5,
trainable=trainable_encoder)
self.conv2 = Conv2D(filters=128,
kernel_size=4,
strides=2,
trainable=trainable_encoder)
self.bne2 = BatchNormalization(momentum=0.9,
epsilon=1e-5,
trainable=trainable_encoder)
if self.res == 64:
self.conv3 = Conv2D(filters=256,
kernel_size=4,
strides=2,
trainable=trainable_encoder)
self.bne3 = BatchNormalization(momentum=0.9,
epsilon=1e-5,
trainable=trainable_encoder)
self.flatten = Flatten()
self.fce1 = Dense(units=1024, trainable=trainable_encoder)
self.bne4 = BatchNormalization(momentum=0.9,
epsilon=1e-5,
trainable=trainable_encoder)
self.fce2 = Dense(units=2 * self.n_z, trainable=trainable_encoder)
# Latent space
self.mean_params = Lambda(lambda x: x[:, :self.n_z])
self.stddev_params = Lambda(lambda x: x[:, self.n_z:])
# Decoder architecture
self.fcd1 = Dense(units=1024, trainable=self.trainable_decoder[0])
self.bnd1 = BatchNormalization(momentum=0.9,
epsilon=1e-5,
trainable=self.trainable_decoder[0])
self.fcd2 = Dense(units=128 * 7 * 7,
trainable=self.trainable_decoder[1])
self.bnd2 = BatchNormalization(momentum=0.9,
epsilon=1e-5,
trainable=self.trainable_decoder[1])
self.reshape = Reshape((7, 7, 128))
if self.res == 64:
self.deconv1 = Conv2DTranspose(filters=128,
kernel_size=4,
strides=2,
padding='valid',
trainable=self.trainable_decoder[2])
self.bnd3 = BatchNormalization(momentum=0.9,
epsilon=1e-5,
trainable=self.trainable_decoder[2])
self.deconv2 = Conv2DTranspose(filters=64,
kernel_size=4,
strides=2,
padding='same',
trainable=self.trainable_decoder[3])
self.bnd4 = BatchNormalization(momentum=0.9,
epsilon=1e-5,
trainable=self.trainable_decoder[3])
self.deconv3 = Conv2DTranspose(filters=1,
kernel_size=4,
strides=2,
padding='same',
activation=self.final_activation,
trainable=self.trainable_decoder[4])
def build_model(self, predict, custom_batch_size=None):
conf = self.conf
model_conf = conf['model']
rnn_size = model_conf['rnn_size']
rnn_type = model_conf['rnn_type']
regularization = model_conf['regularization']
dense_regularization = model_conf['dense_regularization']
use_batch_norm = False
if 'use_batch_norm' in model_conf:
use_batch_norm = model_conf['use_batch_norm']
dropout_prob = model_conf['dropout_prob']
length = model_conf['length']
pred_length = model_conf['pred_length']
# skip = model_conf['skip']
stateful = model_conf['stateful']
return_sequences = model_conf['return_sequences']
# model_conf['output_activation']
output_activation = conf['data']['target'].activation
use_signals = conf['paths']['use_signals']
num_signals = sum([sig.num_channels for sig in use_signals])
num_conv_filters = model_conf['num_conv_filters']
# num_conv_layers = model_conf['num_conv_layers']
size_conv_filters = model_conf['size_conv_filters']
pool_size = model_conf['pool_size']
dense_size = model_conf['dense_size']
batch_size = self.conf['training']['batch_size']
if predict:
batch_size = self.conf['model']['pred_batch_size']
# so we can predict with one time point at a time!
if return_sequences:
length = pred_length
else:
length = 1
if custom_batch_size is not None:
batch_size = custom_batch_size
if rnn_type == 'LSTM':
rnn_model = LSTM
elif rnn_type == 'CuDNNLSTM':
rnn_model = CuDNNLSTM
elif rnn_type == 'SimpleRNN':
rnn_model = SimpleRNN
else:
print('Unkown Model Type, exiting.')
exit(1)
batch_input_shape = (batch_size, length, num_signals)
# batch_shape_non_temporal = (batch_size, num_signals)
indices_0d, indices_1d, num_0D, num_1D = self.get_0D_1D_indices()
def slicer(x, indices):
return x[:, indices]
def slicer_output_shape(input_shape, indices):
shape_curr = list(input_shape)
assert len(shape_curr) == 2 # only valid for 3D tensors
shape_curr[-1] = len(indices)
return tuple(shape_curr)
pre_rnn_input = Input(shape=(num_signals, ))
if num_1D > 0:
pre_rnn_1D = Lambda(
lambda x: x[:, len(indices_0d):],
output_shape=(len(indices_1d), ))(pre_rnn_input)
pre_rnn_0D = Lambda(
lambda x: x[:, :len(indices_0d)],
output_shape=(len(indices_0d), ))(pre_rnn_input)
# slicer(x,indices_0d),lambda s:
# slicer_output_shape(s,indices_0d))(pre_rnn_input)
pre_rnn_1D = Reshape(
(num_1D, len(indices_1d) // num_1D))(pre_rnn_1D)
pre_rnn_1D = Permute((2, 1))(pre_rnn_1D)
if ('simple_conv' in model_conf.keys()
and model_conf['simple_conv'] is True):
for i in range(model_conf['num_conv_layers']):
pre_rnn_1D = Convolution1D(num_conv_filters,
size_conv_filters,
padding='valid',
activation='relu')(pre_rnn_1D)
pre_rnn_1D = MaxPooling1D(pool_size)(pre_rnn_1D)
else:
for i in range(model_conf['num_conv_layers']):
div_fac = 2**i
'''The first conv layer learns `num_conv_filters//div_fac`
filters (aka kernels), each of size
`(size_conv_filters, num1D)`. Its output will have shape
(None, len(indices_1d)//num_1D - size_conv_filters + 1,
num_conv_filters//div_fac), i.e., for
each position in the input spatial series (direction along
radius), the activation of each filter at that position.
'''
'''For i=1 first conv layer would get:
(None, (len(indices_1d)//num_1D - size_conv_filters
+ 1)/pool_size-size_conv_filters + 1,
num_conv_filters//div_fac)
'''
pre_rnn_1D = Convolution1D(num_conv_filters // div_fac,
size_conv_filters,
padding='valid')(pre_rnn_1D)
if use_batch_norm:
pre_rnn_1D = BatchNormalization()(pre_rnn_1D)
pre_rnn_1D = Activation('relu')(pre_rnn_1D)
'''The output of the second conv layer will have shape
(None, len(indices_1d)//num_1D - size_conv_filters + 1,
num_conv_filters//div_fac),
i.e., for each position in the input spatial series
(direction along radius), the activation of each filter
at that position.
For i=1, the second layer would output
(None, (len(indices_1d)//num_1D - size_conv_filters + 1)/
pool_size-size_conv_filters + 1,num_conv_filters//div_fac)
'''
pre_rnn_1D = Convolution1D(num_conv_filters // div_fac,
1,
padding='valid')(pre_rnn_1D)
if use_batch_norm:
pre_rnn_1D = BatchNormalization()(pre_rnn_1D)
pre_rnn_1D = Activation('relu')(pre_rnn_1D)
'''Outputs (None, (len(indices_1d)//num_1D - size_conv_filters
+ 1)/pool_size, num_conv_filters//div_fac)
For i=1, the pooling layer would output:
(None,((len(indices_1d)//num_1D- size_conv_filters
+ 1)/pool_size-size_conv_filters+1)/pool_size,
num_conv_filters//div_fac)
'''
pre_rnn_1D = MaxPooling1D(pool_size)(pre_rnn_1D)
pre_rnn_1D = Flatten()(pre_rnn_1D)
pre_rnn_1D = Dense(
dense_size,
kernel_regularizer=l2(dense_regularization),
bias_regularizer=l2(dense_regularization),
activity_regularizer=l2(dense_regularization))(pre_rnn_1D)
if use_batch_norm:
pre_rnn_1D = BatchNormalization()(pre_rnn_1D)
pre_rnn_1D = Activation('relu')(pre_rnn_1D)
pre_rnn_1D = Dense(
dense_size // 4,
kernel_regularizer=l2(dense_regularization),
bias_regularizer=l2(dense_regularization),
activity_regularizer=l2(dense_regularization))(pre_rnn_1D)
if use_batch_norm:
pre_rnn_1D = BatchNormalization()(pre_rnn_1D)
pre_rnn_1D = Activation('relu')(pre_rnn_1D)
pre_rnn = Concatenate()([pre_rnn_0D, pre_rnn_1D])
else:
pre_rnn = pre_rnn_input
if model_conf['rnn_layers'] == 0 or (
'extra_dense_input' in model_conf.keys()
and model_conf['extra_dense_input']):
pre_rnn = Dense(
dense_size,
activation='relu',
kernel_regularizer=l2(dense_regularization),
bias_regularizer=l2(dense_regularization),
activity_regularizer=l2(dense_regularization))(pre_rnn)
pre_rnn = Dense(
dense_size // 2,
activation='relu',
kernel_regularizer=l2(dense_regularization),
bias_regularizer=l2(dense_regularization),
activity_regularizer=l2(dense_regularization))(pre_rnn)
pre_rnn = Dense(
dense_size // 4,
activation='relu',
kernel_regularizer=l2(dense_regularization),
bias_regularizer=l2(dense_regularization),
activity_regularizer=l2(dense_regularization))(pre_rnn)
pre_rnn_model = tf.keras.Model(inputs=pre_rnn_input, outputs=pre_rnn)
# TODO(KGF): uncomment following lines to get summary of pre-RNN model
# from mpi4py import MPI
# comm = MPI.COMM_WORLD
# task_index = comm.Get_rank()
# if not predict and task_index == 0:
# print('Printing out pre_rnn model...')
# fr = open('model_architecture.log', 'w')
# ori = sys.stdout
# sys.stdout = fr
# pre_rnn_model.summary()
# sys.stdout = ori
# fr.close()
# pre_rnn_model.summary()
x_input = Input(batch_shape=batch_input_shape)
# TODO(KGF): Ge moved this inside a new conditional in Dec 2019. check
# x_in = TimeDistributed(pre_rnn_model)(x_input)
if (num_1D > 0 or ('extra_dense_input' in model_conf.keys()
and model_conf['extra_dense_input'])):
x_in = TimeDistributed(pre_rnn_model)(x_input)
else:
x_in = x_input
# ==========
# TCN MODEL
# ==========
if ('keras_tcn' in model_conf.keys()
and model_conf['keras_tcn'] is True):
print('Building TCN model....')
tcn_layers = model_conf['tcn_layers']
tcn_dropout = model_conf['tcn_dropout']
nb_filters = model_conf['tcn_hidden']
kernel_size = model_conf['kernel_size_temporal']
nb_stacks = model_conf['tcn_nbstacks']
use_skip_connections = model_conf['tcn_skip_connect']
activation = model_conf['tcn_activation']
use_batch_norm = model_conf['tcn_batch_norm']
for _ in range(model_conf['tcn_pack_layers']):
x_in = TCN(use_batch_norm=use_batch_norm,
activation=activation,
use_skip_connections=use_skip_connections,
nb_stacks=nb_stacks,
kernel_size=kernel_size,
nb_filters=nb_filters,
num_layers=tcn_layers,
dropout_rate=tcn_dropout)(x_in)
x_in = Dropout(dropout_prob)(x_in)
else: # end TCN model
# ==========
# RNN MODEL
# ==========
# LSTM in ONNX: "The maximum opset needed by this model is only 9."
model_kwargs = dict(
return_sequences=return_sequences,
# batch_input_shape=batch_input_shape,
stateful=stateful,
kernel_regularizer=l2(regularization),
recurrent_regularizer=l2(regularization),
bias_regularizer=l2(regularization),
)
if rnn_type != 'CuDNNLSTM':
# Dropout is unsupported in CuDNN library
model_kwargs['dropout'] = dropout_prob
model_kwargs['recurrent_dropout'] = dropout_prob
for _ in range(model_conf['rnn_layers']):
x_in = rnn_model(rnn_size, **model_kwargs)(x_in)
x_in = Dropout(dropout_prob)(x_in)
if return_sequences:
# x_out = TimeDistributed(Dense(100,activation='tanh')) (x_in)
x_out = TimeDistributed(Dense(
1, activation=output_activation))(x_in)
model = tf.keras.Model(inputs=x_input, outputs=x_out)
# bug with tensorflow/Keras
# TODO(KGF): what is this bug? this is the only direct "tensorflow"
# import outside of mpi_runner.py and runner.py
# if (conf['model']['backend'] == 'tf'
# or conf['model']['backend'] == 'tensorflow'):
# first_time = "tensorflow" not in sys.modules
# import tensorflow as tf
# if first_time:
# tf.compat.v1.keras.backend.get_session().run(
# tf.global_variables_initializer())
model.reset_states()
return model
def call(self, inputs, training=None, mask=None):
inp = inputs['features_input']
x = conv_bn_pool(
inp,
layer_idx=1,
conv_filters=96,
conv_kernel_size=(7, 7),
conv_strides=(2, 2),
conv_pad=(1, 1),
pool='max',
pool_size=(3, 3),
pool_strides=(2, 2),
)
x = conv_bn_pool(
x,
layer_idx=2,
conv_filters=256,
conv_kernel_size=(5, 5),
conv_strides=(2, 2),
conv_pad=(1, 1),
pool='max',
pool_size=(3, 3),
pool_strides=(2, 2),
)
x = conv_bn_pool(
x,
layer_idx=3,
conv_filters=384,
conv_kernel_size=(3, 3),
conv_strides=(1, 1),
conv_pad=(1, 1),
)
x = conv_bn_pool(
x,
layer_idx=4,
conv_filters=256,
conv_kernel_size=(3, 3),
conv_strides=(1, 1),
conv_pad=(1, 1),
)
x = conv_bn_pool(
x,
layer_idx=5,
conv_filters=256,
conv_kernel_size=(3, 3),
conv_strides=(1, 1),
conv_pad=(1, 1),
pool='max',
pool_size=(5, 3),
pool_strides=(3, 2),
)
x = conv_bn_dynamic_apool(
x,
layer_idx=6,
conv_filters=4096,
conv_kernel_size=(9, 1),
conv_strides=(1, 1),
conv_pad=(0, 0),
conv_layer_prefix='fc',
)
x = conv_bn_pool(
x,
layer_idx=7,
conv_filters=1024,
conv_kernel_size=(1, 1),
conv_strides=(1, 1),
conv_pad=(0, 0),
conv_layer_prefix='fc',
)
x = Lambda(lambda y: K.l2_normalize(y, axis=3), name='norm')(x)
x = Conv2D(
filters=1024,
kernel_size=(1, 1),
strides=(1, 1),
padding='valid',
name='fc8',
)(x)
return x
def create_model_cnn(data_type,
layer_nb=5,
learning_rate=0.0001,
n_label=len(labels),
dense_units=4096):
model = tf.keras.Sequential(name='cnn_' + data_type)
if data_type == 'mfcc':
input_shape = (98, 13)
if data_type == 'ssc':
input_shape = (98, 26)
# Convolution Blocks
# Block 1
model.add(
Conv1D(64,
3,
padding='same',
name='block1_conv',
input_shape=input_shape))
model.add(Lambda(lambda x: K.l2_normalize(x, axis=1)))
model.add(BatchNormalization(name='block1_batchnorm'))
model.add(tf.keras.layers.Activation('relu'))
model.add(MaxPooling1D(2, strides=2, name='block1_pool'))
# Block 2
model.add(Conv1D(128, 3, padding='same', name='block2_conv'))
model.add(Lambda(lambda x: K.l2_normalize(x, axis=1)))
model.add(BatchNormalization(name='block2_batchnorm'))
model.add(tf.keras.layers.Activation('relu'))
model.add(MaxPooling1D(2, strides=2, name='block2_pool'))
# Block 3
model.add(Conv1D(256, 3, padding='same', name='block3_conv'))
model.add(Lambda(lambda x: K.l2_normalize(x, axis=1)))
model.add(BatchNormalization(name='block3_batchnorm'))
model.add(tf.keras.layers.Activation('relu'))
model.add(MaxPooling1D(2, strides=2, name='block3_pool'))
# Block 4
model.add(Conv1D(512, 3, padding='same', name='block4_conv'))
model.add(Lambda(lambda x: K.l2_normalize(x, axis=1)))
model.add(BatchNormalization(name='block4_batchnorm'))
model.add(tf.keras.layers.Activation('relu'))
model.add(MaxPooling1D(2, strides=2, name='block4_pool'))
# Block 5
model.add(Conv1D(512, 3, padding='same', name='block5_conv'))
model.add(Lambda(lambda x: K.l2_normalize(x, axis=1)))
model.add(BatchNormalization(name='block5_batchnorm'))
model.add(tf.keras.layers.Activation('relu'))
model.add(MaxPooling1D(2, strides=2, name='block5_pool'))
# End convolution
model.add(Dropout(0.5, name='Dropout'))
model.add(Flatten(name='flatten'))
# Two Dense layers
model.add(Dense(dense_units, name='fc1'))
model.add(Lambda(lambda x: K.l2_normalize(x, axis=1)))
model.add(BatchNormalization(name='block_batchnorm_dense1'))
model.add(tf.keras.layers.Activation('relu'))
model.add(Dropout(0.5))
model.add(Dense(dense_units, name='fc2'))
model.add(Lambda(lambda x: K.l2_normalize(x, axis=1)))
model.add(BatchNormalization(name='block_batchnorm_dense2'))
model.add(tf.keras.layers.Activation('relu'))
model.add(Dense(n_label, activation='softmax', name='predictions'))
optimizer = RMSprop(lr=learning_rate)
model.compile(loss='categorical_crossentropy',
optimizer=optimizer,
metrics=['accuracy'])
model.summary()
return model
def mlp(
input_shape: Tuple[int, ...], # [28,28]
output_shape: Tuple[int, ...], # (80,)
layer_size: int = 128,
dropout_amount: float = 0.2,
num_layers: int = 3) -> Model:
"""
Simple multi-layer perceptron: just fully-connected layers with dropout between them, with softmax predictions.
Creates num_layers layers.
"""
num_classes = output_shape[0]
model = Sequential()
# Don't forget to pass input_shape to the first layer of the model
##### Your code below (Lab 1)
# model.add(Flatten(input_shape=input_shape))
# for _ in range(num_layers):
# model.add(Dense(layer_size, activation= 'relu'))
# model.add(Dropout(dropout_amount))
# model.add(Dense(num_classes,activation='softmax'))
##### Your code above (Lab 1)
##### CNN Network #2:
# need to add an extra dimension to the input, so that I have 4D.
if len(input_shape) < 3:
model.add(
Lambda(lambda x: tf.expand_dims(x, -1), input_shape=input_shape))
input_shape = (input_shape[0], input_shape[1], 1)
model.add(Conv2D(32, (3, 3), input_shape=(
28, 28, 1))) # this applies 32 convolution filters of size 3x3 each.
model.add(Activation('relu'))
BatchNormalization(axis=-1)
model.add(Conv2D(32, (3, 3)))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
BatchNormalization(axis=-1)
model.add(Conv2D(64, (3, 3)))
model.add(Activation('relu'))
BatchNormalization(axis=-1)
model.add(Conv2D(64, (3, 3)))
model.add(Activation('relu'))
BatchNormalization(axis=-1)
model.add(Conv2D(64, (3, 3)))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Flatten())
# Fully connected layer
BatchNormalization()
model.add(Dense(512))
model.add(Activation('relu'))
BatchNormalization()
model.add(Dropout(dropout_amount))
model.add(Dense(num_classes, activation='softmax'))
return model
def create_model_lstm(data_type,
cnn=False,
layer_nb=5,
classes=len(labels),
lstm_nodes=64,
learning_rate=0.0001):
if data_type == 'mfcc':
input_shape = (98, 13) if not cnn else (98, 13, 1)
if data_type == 'ssc':
input_shape = (98, 26) if not cnn else (98, 26, 1)
model = Sequential(name='lstm_{}'.format('' if not cnn else 'cnn_') +
data_type)
model.add(Input(shape=input_shape))
if cnn:
cnn = Sequential(name='cnn_entry_' + data_type)
cnn.add(Conv1D(22, 3, padding='same', name='conv1'))
cnn.add(BatchNormalization(name='batch_norm1'))
cnn.add(tf.keras.layers.Activation('relu'))
cnn.add(Conv1D(44, 3, padding='same', name='conv2'))
cnn.add(BatchNormalization(name='batch_norm2'))
cnn.add(tf.keras.layers.Activation('relu'))
cnn.add(Conv1D(22, 3, padding='same', name='conv3'))
cnn.add(BatchNormalization(name='batch_norm3'))
cnn.add(tf.keras.layers.Activation('relu'))
cnn.add(AveragePooling1D(2, strides=2, name='pooling'))
model.add(TimeDistributed(cnn))
model.add(Reshape((98, -1)))
if layer_nb == 1:
model.add(LSTM(lstm_nodes, name='lstm_entry'))
model.add(Lambda(lambda x: K.l2_normalize(x, axis=1)))
model.add(BatchNormalization(name='block1_batchnorm'))
else:
model.add(LSTM(lstm_nodes, return_sequences=True, name='lstm_entry'))
model.add(Lambda(lambda x: K.l2_normalize(x, axis=1)))
model.add(BatchNormalization(name='block1_batchnorm'))
for i in range(2, layer_nb):
model.add(
LSTM(lstm_nodes,
return_sequences=True,
name='lstm_{}'.format(i)))
model.add(Lambda(lambda x: K.l2_normalize(x, axis=1)))
model.add(
BatchNormalization(name='block{}_batchnorm'.format(str(i))))
model.add(LSTM(lstm_nodes, name='lstm_out'))
model.add(Lambda(lambda x: K.l2_normalize(x, axis=1)))
model.add(BatchNormalization(name='blockout_batchnorm'))
model.add(Flatten())
model.add(Dense(classes, activation='softmax', name='predictions'))
optimizer = RMSprop(lr=learning_rate)
model.compile(loss='categorical_crossentropy',
optimizer=optimizer,
metrics=['accuracy'])
model.summary()
return model
def discriminator(architecture_size='small',
phaseshuffle_samples=0,
n_classes=5):
discriminator_filters = [64, 128, 256, 512, 1024, 2048]
if architecture_size == 'large':
audio_input_dim = 65536
elif architecture_size == 'medium':
audio_input_dim = 32768
elif architecture_size == 'small':
audio_input_dim = 16384
label_input = Input(shape=(1, ),
dtype='int32',
name='discriminator_label_input')
label_em = Embedding(n_classes, n_classes * 20)(label_input)
label_em = Dense(audio_input_dim)(label_em)
label_em = Reshape((audio_input_dim, 1))(label_em)
discriminator_input = Input(shape=(audio_input_dim, 1),
name='discriminator_input')
x = Concatenate()([discriminator_input, label_em])
if architecture_size == 'small':
# layers 0 to 3
for i in range(4):
x = Conv1D(filters=discriminator_filters[i],
kernel_size=25,
strides=4,
padding='same',
name=f'discriminator_conv_{i}')(x)
x = LeakyReLU(alpha=0.2)(x)
if phaseshuffle_samples > 0:
x = Lambda(apply_phaseshuffle)([x, phaseshuffle_samples])
#layer 4, no phase shuffle
x = Conv1D(filters=discriminator_filters[4],
kernel_size=25,
strides=4,
padding='same',
name=f'discriminator_conv_4')(x)
x = Flatten()(x)
if architecture_size == 'medium':
# layers
for i in range(4):
x = Conv1D(filters=discriminator_filters[i],
kernel_size=25,
strides=4,
padding='same',
name=f'discriminator_conv_{i}')(x)
x = LeakyReLU(alpha=0.2)(x)
if phaseshuffle_samples > 0:
x = Lambda(apply_phaseshuffle)([x, phaseshuffle_samples])
x = Conv1D(filters=discriminator_filters[4],
kernel_size=25,
strides=4,
padding='same',
name='discriminator_conv_4')(x)
x = LeakyReLU(alpha=0.2)(x)
x = Conv1D(filters=discriminator_filters[5],
kernel_size=25,
strides=2,
padding='same',
name='discriminator_conv_5')(x)
x = LeakyReLU(alpha=0.2)(x)
x = Flatten()(x)
if architecture_size == 'large':
# layers
for i in range(4):
x = Conv1D(filters=discriminator_filters[i],
kernel_size=25,
strides=4,
padding='same',
name=f'discriminator_conv_{i}')(x)
x = LeakyReLU(alpha=0.2)(x)
if phaseshuffle_samples > 0:
x = Lambda(apply_phaseshuffle)([x, phaseshuffle_samples])
#last 2 layers without phase shuffle
x = Conv1D(filters=discriminator_filters[4],
kernel_size=25,
strides=4,
padding='same',
name='discriminator_conv_4')(x)
x = LeakyReLU(alpha=0.2)(x)
x = Conv1D(filters=discriminator_filters[5],
kernel_size=25,
strides=4,
padding='same',
name='discriminator_conv_5')(x)
x = LeakyReLU(alpha=0.2)(x)
x = Flatten()(x)
discriminator_output = Dense(1)(x)
discriminator = Model([discriminator_input, label_input],
discriminator_output,
name='Discriminator')
return discriminator
def _build_model(self, x, y):
"""Construct ASAC model using feature and label statistics.
Args:
- x: temporal feature
- y: labels
Returns:
- model: asac model
"""
# Parameters
h_dim = self.h_dim
n_layer = self.n_layer
dim = len(x[0, 0, :])
max_seq_len = len(x[0, :, 0])
# Build one input, two outputs model
main_input = Input(shape=(max_seq_len, dim), dtype='float32')
mask_layer = Masking(mask_value=-1.)(main_input)
previous_input = Input(shape=(max_seq_len, dim), dtype='float32')
previous_mask_layer = Masking(mask_value=-1.)(previous_input)
select_layer = rnn_layer(previous_mask_layer,
self.model_type,
h_dim,
return_seq=True)
for _ in range(n_layer):
select_layer = rnn_layer(select_layer,
self.model_type,
h_dim,
return_seq=True)
select_layer = TimeDistributed(Dense(
dim, activation='sigmoid'))(select_layer)
# Sampling the selection
select_layer = Lambda(lambda x: x - 0.5)(select_layer)
select_layer = Activation('relu')(select_layer)
select_out = Lambda(lambda x: x * 2, name='select')(select_layer)
# Second output
pred_layer = Multiply()([mask_layer, select_out])
for _ in range(n_layer - 1):
pred_layer = rnn_layer(pred_layer,
self.model_type,
h_dim,
return_seq=True)
return_seq_bool = len(y.shape) == 3
pred_layer = rnn_layer(pred_layer, self.model_type, h_dim,
return_seq_bool)
if self.task == 'classification': act_fn = 'sigmoid'
elif self.task == 'regression': act_fn = 'linear'
if len(y.shape) == 3:
pred_out = TimeDistributed(Dense(y.shape[-1], activation=act_fn),
name='predict')(pred_layer)
elif len(y.shape) == 2:
pred_out = Dense(y.shape[-1], activation=act_fn,
name='predict')(pred_layer)
model = Model(inputs=[main_input, previous_input],
outputs=[select_out, pred_out])
# Optimizer
adam = tf.keras.optimizers.Adam(learning_rate=self.learning_rate,
beta_1=0.9,
beta_2=0.999,
amsgrad=False)
# Model compile
if self.task == 'classification':
model.compile(loss={
'select': select_loss,
'predict': binary_cross_entropy_loss
},
optimizer=adam,
loss_weights={
'select': 0.01,
'predict': 1
})
elif self.task == 'regression':
model.compile(loss={
'select': select_loss,
'predict': rmse_loss
},
optimizer=adam,
loss_weights={
'select': 0.01,
'predict': 1
})
return model
conv_4 = Conv2D(256, (3, 3), activation='relu', padding='same')(conv_3)
# poolig layer with kernel size (2,1)
pool_4 = MaxPool2D(pool_size=(2, 1))(conv_4)
conv_5 = Conv2D(512, (3, 3), activation='relu', padding='same')(pool_4)
# Batch normalization layer
batch_norm_5 = BatchNormalization()(conv_5)
conv_6 = Conv2D(512, (3, 3), activation='relu', padding='same')(batch_norm_5)
batch_norm_6 = BatchNormalization()(conv_6)
pool_6 = MaxPool2D(pool_size=(2, 1))(batch_norm_6)
conv_7 = Conv2D(512, (2, 2), activation='relu')(pool_6)
squeezed = Lambda(lambda x: K.squeeze(x, 1))(conv_7)
# bidirectional LSTM layers with units=128
blstm_1 = Bidirectional(LSTM(128, return_sequences=True,
dropout=0.2))(squeezed)
blstm_2 = Bidirectional(LSTM(128, return_sequences=True, dropout=0.2))(blstm_1)
outputs = Dense(len(char_list) + 1, activation='softmax')(blstm_2)
# model to be used at test time
act_model = Model(inputs, outputs)
act_model.summary()
#%%
labels = Input(name='the_labels', shape=[max_label_len], dtype='float32')
input_length = Input(name='input_length', shape=[1], dtype='int64')
def __init__(
self,
layer_sizes,
generator=None,
aggregator=None,
bias=True,
dropout=0.0,
normalize="l2",
activations=None,
**kwargs,
):
# Model parameters
self.layer_sizes = layer_sizes
self.max_hops = len(layer_sizes)
self.bias = bias
self.dropout = dropout
# Set the normalization layer used in the model
if normalize == "l2":
self._normalization = Lambda(lambda x: K.l2_normalize(x, axis=-1))
elif normalize is None or normalize == "none" or normalize == "None":
self._normalization = Lambda(lambda x: x)
else:
raise ValueError(
"Normalization should be either 'l2' or 'none'; received '{}'".format(
normalize
)
)
# Get the input_dim and num_samples
if generator is not None:
self._get_sizes_from_generator(generator)
else:
self._get_sizes_from_keywords(kwargs)
# Feature dimensions for each layer
self.dims = [self.input_feature_size] + layer_sizes
# Compute size of each sampled neighbourhood
self._compute_neighbourhood_sizes()
# Set the aggregator layer used in the model
if aggregator is None:
self._aggregator = MeanAggregator
elif issubclass(aggregator, Layer):
self._aggregator = aggregator
else:
raise TypeError("Aggregator should be a subclass of Keras Layer")
# Activation function for each layer
if activations is None:
activations = ["relu"] * (self.max_hops - 1) + ["linear"]
elif len(activations) != self.max_hops:
raise ValueError(
"Invalid number of activations; require one function per layer"
)
self.activations = activations
# Optional regulariser, etc. for weights and biases
self._get_regularisers_from_keywords(kwargs)
# Aggregator functions for each layer
self._build_aggregators()
def __init__(self, config, latent_code_garms_sz=1024, garmparams_sz=config.PCA_, name=None):
super(PoseShapeOffsetModel, self).__init__(name=name)
self.config = config
self.latent_code_garms_sz = latent_code_garms_sz
self.garmparams_sz = garmparams_sz
self.latent_code_betas_sz = 128
##ToDo: Minor: Remove hard coded colors. Should be same as rendered colors in input
self.colormap = tf.cast(
[np.array([255, 255, 255]), np.array([65, 0, 65]), np.array([0, 65, 65]), np.array([145, 65, 0]),
np.array([145, 0, 65]),
np.array([0, 145, 65])], tf.float32) / 255.
hres_file = open('assets/hresMapping.pkl', 'rb')
u = pkl._Unpickler(hres_file)
u.encoding = 'latin1'
_, self.faces = u.load()
self.faces = np.int32(self.faces)
## Define network layers
self.top_ = SingleImageNet(self.latent_code_garms_sz, self.latent_code_betas_sz)
for n in self.config.garmentKeys:
gn = GarmentNet(self.config.PCA_, n, self.garmparams_sz)
self.garmentModels.append(gn)
self.smpl = SMPL('assets/neutral_smpl.pkl',
theta_in_rodrigues=False, theta_is_perfect_rotmtx=False, isHres=True, scale=True)
self.smpl_J = SmplBody25Layer(theta_in_rodrigues=False, theta_is_perfect_rotmtx=False, isHres=True)
self.J_layers = [NameLayer('J_{}'.format(i)) for i in range(NUM)]
self.lat_betas = Dense(self.latent_code_betas_sz, kernel_initializer=initializers.RandomNormal(0, 0.00005),
activation='relu')
self.betas = Dense(10, kernel_initializer=initializers.RandomNormal(0, 0.000005), name='betas')
init_trans = np.array([0, 0.2, -2.])
init_pose = np.load('assets/mean_a_pose.npy')
init_pose[:3] = 0
init_pose = tf.reshape(batch_rodrigues(init_pose.reshape(-1, 3).astype(np.float32)), (-1,))
self.pose_trans = tf.concat((init_pose, init_trans), axis=0)
self.lat_pose = Dense(self.latent_code_betas_sz, kernel_initializer=initializers.RandomNormal(0, 0.000005),
activation='relu')
self.lat_pose_layer = Dense(24 * 3 * 3 + 3, kernel_initializer=initializers.RandomNormal(0, 0.000005),
name='pose_trans')
self.cut_trans = Lambda(lambda z: z[:, -3:])
self.trans_layers = [NameLayer('trans_{}'.format(i)) for i in range(NUM)]
self.cut_poses = Lambda(lambda z: z[:, :-3])
self.reshape_pose = Reshape((24, 3, 3))
self.pose_layers = [NameLayer('pose_{}'.format(i)) for i in range(NUM)]
## Optional: Condition garment on betas, probably not
self.latent_code_offset_ShapeMerged = Dense(self.latent_code_garms_sz + self.latent_code_betas_sz,
activation='relu')
self.latent_code_offset_ShapeMerged_2 = Dense(self.latent_code_garms_sz + self.latent_code_betas_sz,
activation='relu', name='latent_code_offset_ShapeMerged')
self.avg = Average()
self.flatten = Flatten()
self.concat = Concatenate()
self.scatters = []
for vs in self.vertSpread:
self.scatters.append(Scatter_(vs, self.config.NVERTS))
def __init__(self, input_size=512):
input_image = Input(shape=(None, None, 3), name='input_image')
overly_small_text_region_training_mask = Input(
shape=(None, None, 1),
name='overly_small_text_region_training_mask')
text_region_boundary_training_mask = Input(
shape=(None, None, 1), name='text_region_boundary_training_mask')
target_score_map = Input(shape=(None, None, 1),
name='target_score_map')
resnet = ResNet50(input_tensor=input_image,
weights='imagenet',
include_top=False,
pooling=None)
#x = resnet.get_layer('activation_49').output
x = resnet.get_layer('conv5_block3_out').output
x = Lambda(resize_bilinear, name='resize_1')(x)
# x = concatenate([x, resnet.get_layer('activation_40').output], axis=3)
x = concatenate([x, resnet.get_layer('conv4_block6_out').output],
axis=3)
x = Conv2D(128, (1, 1),
padding='same',
kernel_regularizer=regularizers.l2(1e-5))(x)
x = BatchNormalization(momentum=0.997, epsilon=1e-5, scale=True)(x)
x = Activation('relu')(x)
x = Conv2D(128, (3, 3),
padding='same',
kernel_regularizer=regularizers.l2(1e-5))(x)
x = BatchNormalization(momentum=0.997, epsilon=1e-5, scale=True)(x)
x = Activation('relu')(x)
x = Lambda(resize_bilinear, name='resize_2')(x)
# x = concatenate([x, resnet.get_layer('activation_22').output], axis=3)
x = concatenate([x, resnet.get_layer('conv3_block4_out').output],
axis=3)
x = Conv2D(64, (1, 1),
padding='same',
kernel_regularizer=regularizers.l2(1e-5))(x)
x = BatchNormalization(momentum=0.997, epsilon=1e-5, scale=True)(x)
x = Activation('relu')(x)
x = Conv2D(64, (3, 3),
padding='same',
kernel_regularizer=regularizers.l2(1e-5))(x)
x = BatchNormalization(momentum=0.997, epsilon=1e-5, scale=True)(x)
x = Activation('relu')(x)
x = Lambda(resize_bilinear, name='resize_3')(x)
# x = concatenate([x, ZeroPadding2D(((1, 0),(1, 0)))(resnet.get_layer('activation_10').output)], axis=3)
x = concatenate([x, resnet.get_layer('conv2_block3_out').output],
axis=3)
x = Conv2D(32, (1, 1),
padding='same',
kernel_regularizer=regularizers.l2(1e-5))(x)
x = BatchNormalization(momentum=0.997, epsilon=1e-5, scale=True)(x)
x = Activation('relu')(x)
x = Conv2D(32, (3, 3),
padding='same',
kernel_regularizer=regularizers.l2(1e-5))(x)
x = BatchNormalization(momentum=0.997, epsilon=1e-5, scale=True)(x)
x = Activation('relu')(x)
x = Conv2D(32, (3, 3),
padding='same',
kernel_regularizer=regularizers.l2(1e-5))(x)
x = BatchNormalization(momentum=0.997, epsilon=1e-5, scale=True)(x)
x = Activation('relu')(x)
pred_score_map = Conv2D(1, (1, 1),
activation=tf.nn.sigmoid,
name='pred_score_map')(x)
rbox_geo_map = Conv2D(4, (1, 1),
activation=tf.nn.sigmoid,
name='rbox_geo_map')(x)
rbox_geo_map = Lambda(lambda x: x * input_size)(rbox_geo_map)
angle_map = Conv2D(1, (1, 1),
activation=tf.nn.sigmoid,
name='rbox_angle_map')(x)
angle_map = Lambda(lambda x: (x - 0.5) * np.pi / 2)(angle_map)
pred_geo_map = concatenate([rbox_geo_map, angle_map],
axis=3,
name='pred_geo_map')
model = Model(inputs=[
input_image, overly_small_text_region_training_mask,
text_region_boundary_training_mask, target_score_map
],
outputs=[pred_score_map, pred_geo_map])
# print(model.summary())
self.input_image = input_image
self.input_size = input_size
self.overly_small_text_region_training_mask = overly_small_text_region_training_mask
self.text_region_boundary_training_mask = text_region_boundary_training_mask
self.target_score_map = target_score_map
self.pred_score_map = pred_score_map
self.pred_geo_map = pred_geo_map
self.model = model
def __init__(self, latent_code_garms_sz, latent_code_betas_sz, name=None):
super(SingleImageNet, self).__init__(name=name)
## Define layers
if IMG_SIZE >= 1000:
self.conv1 = Conv2D(8, (3, 3), kernel_initializer='he_normal', padding='same', activation='relu')
self.conv1_1 = Conv2D(16, (3, 3), strides=(2, 2), kernel_initializer='he_normal', padding='same',
activation='relu')
else:
self.conv1 = Conv2D(8, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')
self.conv1_1 = Conv2D(16, (3, 3), activation='relu', kernel_initializer='he_normal',
padding='same')
self.conv2 = Conv2D(16, (3, 3), # strides=(2, 2),
kernel_initializer='he_normal', padding='same', activation='relu')
self.conv2_1 = Conv2D(32, (3, 3), # strides=(2, 2),
kernel_initializer='he_normal', padding='same', activation='relu')
self.conv3 = Conv2D(16, (3, 3), # strides=(2, 2),
kernel_initializer='he_normal', padding='same', activation='relu')
self.conv3_1 = Conv2D(32, (3, 3), # strides=(2, 2),
kernel_initializer='he_normal', padding='same', activation='relu')
sz = 16
self.conv4 = Conv2D(sz, (3, 3), # strides=(2, 2),
kernel_initializer='he_normal', padding='same', activation='relu')
self.conv4_1 = Conv2D(sz, (3, 3), # strides=(2, 2),
kernel_initializer='he_normal', padding='same', activation='relu')
self.conv5 = Conv2D(sz, (3, 3), kernel_initializer='he_normal', padding='same',
activation='relu')
self.conv5_1 = Conv2D(sz, (3, 3), kernel_initializer='he_normal', padding='same',
activation='relu')
self.conv6 = Conv2D(sz, (3, 3), kernel_initializer='he_normal', padding='same',
activation='relu')
self.conv6_1 = Conv2D(sz, (3, 3), kernel_initializer='he_normal', padding='same',
activation='relu')
self.conv7 = Conv2D(sz, (3, 3), kernel_initializer='he_normal', padding='same',
activation='relu')
self.conv7_1 = Conv2D(sz, (3, 3), kernel_initializer='he_normal', padding='same',
activation='relu')
self.split_shape = Lambda(lambda z: z[..., :int(sz / 2)])
self.split_garms = Lambda(lambda z: z[..., int(sz / 2):])
self.flatten = Flatten()
self.dg = Dense(latent_code_garms_sz, kernel_initializer='he_normal', activation='relu')
self.db = Dense(latent_code_betas_sz, kernel_initializer='he_normal', activation='relu')
self.dg2 = Dense(int(latent_code_garms_sz / 2), kernel_initializer='he_normal', activation='relu')
self.db2 = Dense(latent_code_betas_sz, kernel_initializer='he_normal', activation='relu')
self.dg3 = Dense(int(latent_code_garms_sz / 2), kernel_initializer='he_normal', activation='relu')
self.db3 = Dense(latent_code_betas_sz, kernel_initializer='he_normal', activation='relu')
self.latent_code_garms = Dense(int(latent_code_garms_sz / 2), kernel_initializer='he_normal',
name='latent_garms', activation='relu')
self.latent_code_shape = Dense(latent_code_betas_sz, kernel_initializer='he_normal', name='latent_shape',
activation='relu')
self.concat = Concatenate()
def _inception_resnet_block(x,
scale,
block_type,
block_idx,
activation='relu'):
channel_axis = 3
if block_idx is None:
prefix = None
else:
prefix = '_'.join((block_type, str(block_idx)))
name_fmt = partial(_generate_layer_name, prefix=prefix)
if block_type == 'Block35':
branch_0 = conv2d_bn(x, 32, 1, name=name_fmt('Conv2d_1x1', 0))
branch_1 = conv2d_bn(x, 32, 1, name=name_fmt('Conv2d_0a_1x1', 1))
branch_1 = conv2d_bn(branch_1,
32,
3,
name=name_fmt('Conv2d_0b_3x3', 1))
branch_2 = conv2d_bn(x, 32, 1, name=name_fmt('Conv2d_0a_1x1', 2))
branch_2 = conv2d_bn(branch_2,
32,
3,
name=name_fmt('Conv2d_0b_3x3', 2))
branch_2 = conv2d_bn(branch_2,
32,
3,
name=name_fmt('Conv2d_0c_3x3', 2))
branches = [branch_0, branch_1, branch_2]
elif block_type == 'Block17':
branch_0 = conv2d_bn(x, 128, 1, name=name_fmt('Conv2d_1x1', 0))
branch_1 = conv2d_bn(x, 128, 1, name=name_fmt('Conv2d_0a_1x1', 1))
branch_1 = conv2d_bn(branch_1,
128, [1, 7],
name=name_fmt('Conv2d_0b_1x7', 1))
branch_1 = conv2d_bn(branch_1,
128, [7, 1],
name=name_fmt('Conv2d_0c_7x1', 1))
branches = [branch_0, branch_1]
elif block_type == 'Block8':
branch_0 = conv2d_bn(x, 192, 1, name=name_fmt('Conv2d_1x1', 0))
branch_1 = conv2d_bn(x, 192, 1, name=name_fmt('Conv2d_0a_1x1', 1))
branch_1 = conv2d_bn(branch_1,
192, [1, 3],
name=name_fmt('Conv2d_0b_1x3', 1))
branch_1 = conv2d_bn(branch_1,
192, [3, 1],
name=name_fmt('Conv2d_0c_3x1', 1))
branches = [branch_0, branch_1]
mixed = Concatenate(axis=channel_axis,
name=name_fmt('Concatenate'))(branches)
up = conv2d_bn(mixed,
K.int_shape(x)[channel_axis],
1,
activation=None,
use_bias=True,
name=name_fmt('Conv2d_1x1'))
up = Lambda(scaling,
output_shape=K.int_shape(up)[1:],
arguments={'scale': scale})(up)
x = add([x, up])
if activation is not None:
x = Activation(activation, name=name_fmt('Activation'))(x)
return x
def create_cam_model(args, input_size=[256, 256]):
"""
Hyper-parameters:
learning_rate: Learning-rate for the optimizer.
num_dense_layers: Number of dense layers.
num_dense_nodes: Number of nodes in each dense layer.
activation: Activation function for all layers.
"""
num_cnn1 = args["num_cnn1"]
num_cnn2 = args["num_cnn2"]
num_cnn3 = args["num_cnn3"]
kernel_size = args["kernel_size"]
# Start construction of a Keras Sequential model.
model = Sequential()
# Add an input layer which is similar to a feed_dict in TensorFlow.
# Note that the input-shape must be a tuple containing the image-size.
model.add(InputLayer(input_shape=input_size))
# The input from MNIST is a flattened array with 784 elements,
# but the convolutional layers expect images with shape (28, 28, 1)
model.add(Reshape(img_shape_full))
# 1st convolutional layer.
# There are many hyper-parameters in this layer, but we only
# want to optimize the activation-function in this example.
model.add(
Conv2D(kernel_size=(kernel_size, 1),
strides=(1, 1),
filters=num_cnn1,
padding='same',
activation="relu",
name='layer_conv1'))
model.add(MaxPooling2D(pool_size=(2, 1), strides=(2, 1)))
# 2nd convolutional layer.
# Again, we only want to optimize the activation-function here.
model.add(
Conv2D(kernel_size=(kernel_size, 1),
strides=(1, 1),
filters=num_cnn2,
padding='same',
activation="relu",
name='layer_conv2'))
model.add(MaxPooling2D(pool_size=(2, 1), strides=(2, 1)))
# 3rd convolutional layer.
# Again, we only want to optimize the activation-function here.
model.add(
Conv2D(kernel_size=(kernel_size, 1),
strides=(1, 1),
filters=num_cnn3,
padding='same',
activation="relu",
name='layer_conv3'))
# Global average pooling
model.add(
Lambda(global_average_pooling,
output_shape=global_average_pooling_shape))
model.add(
Dense(num_classes,
activation='softmax',
kernel_initializer='random_uniform'))
model.summary()
return model
def attention_block(input,
input_channels=None,
output_channels=None,
encoder_depth=1):
p = 1
t = 2
r = 1
if input_channels is None:
input_channels = input.get_shape()[-1]
if output_channels is None:
output_channels = input_channels
# First Residual Block
for i in range(p):
input = residual_block(input)
# Trunc Branch
output_trunk = input
for i in range(t):
output_trunk = residual_block(output_trunk)
# Soft Mask Branch
## encoder
### first down sampling
output_soft_mask = MaxPool2D(padding='same')(input)
for i in range(r):
output_soft_mask = residual_block(output_soft_mask)
skip_connections = []
for i in range(encoder_depth - 1):
## skip connections
output_skip_connection = residual_block(output_soft_mask)
skip_connections.append(output_skip_connection)
# print ('skip shape:', output_skip_connection.get_shape())
## down sampling
output_soft_mask = MaxPool2D(padding='same')(output_soft_mask)
for _ in range(r):
output_soft_mask = residual_block(output_soft_mask)
skip_connections = list(reversed(skip_connections))
for i in range(encoder_depth - 1):
for _ in range(r):
output_soft_mask = residual_block(output_soft_mask)
output_soft_mask = UpSampling2D()(output_soft_mask)
output_soft_mask = Add()([output_soft_mask, skip_connections[i]])
for i in range(r):
output_soft_mask = residual_block(output_soft_mask)
output_soft_mask = UpSampling2D()(output_soft_mask)
## Output
output_soft_mask = Conv2D(input_channels, (1, 1))(output_soft_mask)
output_soft_mask = Conv2D(input_channels, (1, 1))(output_soft_mask)
output_soft_mask = Activation('sigmoid')(output_soft_mask)
# Attention: (1 + output_soft_mask) * output_trunk
output = Lambda(lambda x: x + 1)(output_soft_mask)
output = Multiply()([output, output_trunk]) #
# Last Residual Block
for i in range(p):
output = residual_block(output)
return output
def train(run_name, start_epoch, stop_epoch, img_w):
# Input Parameters
img_h = 64
words_per_epoch = 16000
val_split = 0.2
val_words = int(words_per_epoch * (val_split))
# Network parameters
conv_filters = 16
kernel_size = (3, 3)
pool_size = 2
time_dense_size = 32
rnn_size = 512
minibatch_size = 32
if K.image_data_format() == 'channels_first':
input_shape = (1, img_w, img_h)
else:
input_shape = (img_w, img_h, 1)
fdir = '.'
img_gen = TextImageGenerator(
monogram_file=os.path.join(fdir, 'wordlist_mono_clean.txt'),
bigram_file=os.path.join(fdir, 'wordlist_bi_clean.txt'),
minibatch_size=minibatch_size,
img_w=img_w,
img_h=img_h,
downsample_factor=(pool_size**2),
val_split=words_per_epoch - val_words)
act = 'relu'
input_data = Input(name='the_input', shape=input_shape, dtype='float32')
inner = Conv2D(conv_filters,
kernel_size,
padding='same',
activation=act,
kernel_initializer='he_normal',
name='conv1')(input_data)
inner = MaxPooling2D(pool_size=(pool_size, pool_size), name='max1')(inner)
inner = Conv2D(conv_filters,
kernel_size,
padding='same',
activation=act,
kernel_initializer='he_normal',
name='conv2')(inner)
inner = MaxPooling2D(pool_size=(pool_size, pool_size), name='max2')(inner)
conv_to_rnn_dims = (img_w // (pool_size**2),
(img_h // (pool_size**2)) * conv_filters)
inner = Reshape(target_shape=conv_to_rnn_dims, name='reshape')(inner)
# cuts down input size going into RNN:
inner = Dense(time_dense_size, activation=act, name='dense1')(inner)
# Two layers of bidirectional GRUs
# GRU seems to work as well, if not better than LSTM:
gru_1 = GRU(rnn_size,
return_sequences=True,
kernel_initializer='he_normal',
name='gru1')(inner)
gru_1b = GRU(rnn_size,
return_sequences=True,
go_backwards=True,
kernel_initializer='he_normal',
name='gru1_b')(inner)
gru1_merged = add([gru_1, gru_1b])
gru_2 = GRU(rnn_size,
return_sequences=True,
kernel_initializer='he_normal',
name='gru2')(gru1_merged)
gru_2b = GRU(rnn_size,
return_sequences=True,
go_backwards=True,
kernel_initializer='he_normal',
name='gru2_b')(gru1_merged)
# transforms RNN output to character activations:
inner = Dense(img_gen.get_output_size(),
kernel_initializer='he_normal',
name='dense2')(concatenate([gru_2, gru_2b]))
y_pred = Activation('softmax', name='softmax')(inner)
Model(inputs=input_data, outputs=y_pred).summary()
labels = Input(name='the_labels',
shape=[img_gen.absolute_max_string_len],
dtype='float32')
input_length = Input(name='input_length', shape=[1], dtype='int64')
label_length = Input(name='label_length', shape=[1], dtype='int64')
# Keras doesn't currently support loss funcs with extra parameters
# so CTC loss is implemented in a lambda layer
loss_out = Lambda(ctc_lambda_func, output_shape=(1, ),
name='ctc')([y_pred, labels, input_length, label_length])
# clipnorm seems to speeds up convergence
sgd = SGD(lr=0.02, decay=1e-6, momentum=0.9, nesterov=True, clipnorm=5)
model = Model(inputs=[input_data, labels, input_length, label_length],
outputs=loss_out)
# the loss calc occurs elsewhere, so use a dummy lambda func for the loss
model.compile(loss={
'ctc': lambda y_true, y_pred: y_pred
},
optimizer=sgd,
metrics=['accuracy'])
if start_epoch > 0:
weight_file = os.path.join(
OUTPUT_DIR,
os.path.join(run_name, 'weights%02d.h5' % (start_epoch - 1)))
model.load_weights(weight_file)
# captures output of softmax so we can decode the output during visualization
test_func = K.function([input_data], [y_pred])
viz_cb = VizCallback(run_name, test_func, img_gen.next_val())
model.fit(img_gen.next_train(),
steps_per_epoch=(words_per_epoch - val_words) // minibatch_size,
epochs=stop_epoch,
validation_data=img_gen.next_val(),
validation_steps=val_words // minibatch_size,
callbacks=[viz_cb, img_gen],
initial_epoch=start_epoch)
def centernet(num_classes,
input_size=512,
max_objects=100,
score_threshold=0.1,
nms=True,
flip_test=False,
training=True,
l2_norm=5e-4):
output_size = input_size // 4
image_input = Input(shape=(input_size, input_size, 3))
hm_input = Input(shape=(output_size, output_size, num_classes))
wh_input = Input(shape=(max_objects, 2))
reg_input = Input(shape=(max_objects, 2))
reg_mask_input = Input(shape=(max_objects, ))
index_input = Input(shape=(max_objects, ))
resnet = ResNet50(include_top=False, input_tensor=image_input)
C2, C3, C4, C5 = resnet.outputs
x = C5
x = UpSampling2D()(x)
x = Conv2D(256, (1, 1),
padding='same',
use_bias=False,
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_norm))(x)
x = BatchNormalization()(x)
x = relu(x, max_value=6)
x = Concatenate()([C4, x])
x = Conv2D(256, (3, 3),
padding='same',
use_bias=False,
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_norm))(x)
x = BatchNormalization()(x)
x = relu(x, max_value=6)
# (b, 32, 32, 512)
x = UpSampling2D()(x)
x = Conv2D(128, (1, 1),
padding='same',
use_bias=False,
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_norm))(x)
x = BatchNormalization()(x)
x = relu(x, max_value=6)
x = Concatenate()([C3, x])
x = Conv2D(128, (3, 3),
padding='same',
use_bias=False,
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_norm))(x)
x = BatchNormalization()(x)
x = relu(x, max_value=6)
# (b, 64, 64, 128)
x = UpSampling2D()(x)
x = Conv2D(64, (1, 1),
padding='same',
use_bias=False,
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_norm))(x)
x = BatchNormalization()(x)
x = relu(x, max_value=6)
x = Concatenate()([C2, x])
x = Conv2D(64, (3, 3),
padding='same',
use_bias=False,
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_norm))(x)
x = BatchNormalization()(x)
x = relu(x, max_value=6)
# hm header
y1 = Conv2D(64, (3, 3),
padding='same',
use_bias=False,
kernel_initializer='he_norm',
kernel_regularizer=l2(l2_norm))(x)
y1 = BatchNormalization()(y1)
y1 = relu(y1, max_value=6)
y1 = Conv2D(num_classes, (1, 1),
kernel_initializer='he_norm',
kernel_regularizer=l2(l2_norm),
activation='sigmoid')(y1)
# wh header
y2 = Conv2D(64, (3, 3),
use_bias=False,
kernel_initializer='he_norm',
kernel_regularizer=l2(l2_norm))(x)
y2 = BatchNormalization()(y2)
y2 = relu(y2, max_value=6)
y2 = Conv2D(2, (1, 1),
kernel_initializer='he_norm',
kernel_regularizer=l2(l2_norm))(y2)
# reg header
y3 = Conv2D(64, (3, 3),
use_bias=False,
kernel_initializer='he_norm',
kernel_regularizer=l2(l2_norm))(x)
y3 = BatchNormalization()(y3)
y3 = relu(y3, max_value=6)
y3 = Conv2D(2, (1, 1),
kernel_initializer='he_norm',
kernel_regularizer=l2(l2_norm))(y3)
loss_ = Lambda(loss, name='centernet_loss')([
y1, y2, y3, hm_input, wh_input, reg_input, reg_mask_input, index_input
])
if training:
model = Model(inputs=[
image_input, hm_input, wh_input, reg_input, reg_mask_input,
index_input
],
outputs=[loss_])
else:
detections = Lambda(lambda x: decode(*x,
max_objects=max_objects,
score_threshold=score_threshold,
nms=nms,
num_classes=num_classes))(
[y1, y2, y3])
model = Model(inputs=image_input, outputs=detections)
return model
