def spatial(self, ind_emb):
kernel_size = 3
kernel_initializer1 = tf.contrib.layers.variance_scaling_initializer()
# with tf.variable_scope(name):
avg_emb = []
max_emb = []
std_emb = []
for emb in ind_emb:
avg_pool = Lambda(lambda x: K.mean(x, axis=[1], keepdims=True))(
emb)
assert avg_pool.get_shape()[1] == 1
avg_emb.append(avg_pool)
max_pool = Lambda(lambda x: K.max(x, axis=[1], keepdims=True))(emb)
assert max_pool.get_shape()[1] == 1
max_emb.append(max_pool)
# std_pool = Lambda(lambda x: K.std(x, axis=[1], keepdims=True))(emb)
# assert std_pool.get_shape()[1] == 1
# std_emb.append(std_pool)
avg_emb = Concatenate(axis=1)(avg_emb)
max_emb = Concatenate(axis=1)(max_emb)
# std_emb = Concatenate(axis=1)(std_emb)
concat = Concatenate(axis=1)([avg_emb, max_emb])
#assert concat.get_shape()[1] == 6
concat = Lambda(lambda x: K.expand_dims(x, 0))(concat)
concat = Lambda(lambda x: K.permute_dimensions(x, (0, 2, 1, 3, 4)))(
concat)
concat = Conv3D(filters=1,
kernel_size=[kernel_size, kernel_size, kernel_size],
strides=[1, 1, 1],
padding="same",
activation=None,
kernel_initializer='random_uniform',
use_bias=False,
name='conv')(concat)
#assert concat.get_shape()[-1] == 1
concat = Activation('sigmoid')(concat)
concat = Lambda(lambda x: K.squeeze(x, 0))(concat)
ind_emb = [
Multiply()([
a,
Lambda(lambda x: K.permute_dimensions(x, (1, 0, 2, 3)))(concat)
]) for a in ind_emb
]
# concat = Lambda(lambda x: K.permute_dimensions(x,(1,0,2,3)))(concat)
# return Lambda(lambda x: x[:,0:4,:,:])(concat)
return Lambda(lambda x: ave(x))(ind_emb)
def unet(x_in, pose_in, nf_enc, nf_dec):
x0 = my_conv(x_in, nf_enc[0], ks=7) # 256
x1 = my_conv(x0, nf_enc[1], strides=2) # 128
x2 = concatenate([x1, pose_in])
x3 = my_conv(x2, nf_enc[2])
x4 = my_conv(x3, nf_enc[3], strides=2) # 64
x5 = my_conv(x4, nf_enc[4])
x6 = my_conv(x5, nf_enc[5], strides=2) # 32
x7 = my_conv(x6, nf_enc[6])
x8 = my_conv(x7, nf_enc[7], strides=2) # 16
x9 = my_conv(x8, nf_enc[8])
x10 = my_conv(x9, nf_enc[9], strides=2) # 8
x = my_conv(x10, nf_enc[10])
skips = [x9, x7, x5, x3, x0]
filters = [nf_enc[10], nf_dec[0], nf_dec[1], nf_dec[2], nf_enc[3]]
for i in range(5):
out_sz = 8 * (2**(i + 1))
print(x.get_shape())
x = Lambda(interp_upsampling,
output_shape=(out_sz, out_sz, filters[i]))(x)
print("@@@@@@@@@@@@@@@@@@@@@@@@@@")
print(i, x.get_shape(), skips[i].get_shape())
print("@@@@@@@@@@@@@@@@@@@@@@@@@@")
x = concatenate([x, skips[i]])
x = my_conv(x, nf_dec[i])
return x
def set_model(self):
'''
Encoder
'''
# define input with 'channels_first'
input_encoder = Input(shape=self.input_shape, name='encoder_input')
conv_1 = Conv2D(1,
kernel_size=self.kernel_size,
padding='same',
activation='relu')(input_encoder)
conv_2 = Conv2D(self.filters,
kernel_size=self.kernel_size,
padding='same',
activation='relu')(conv_1)
conv2_out_shape = tuple(conv_2.get_shape().as_list())
flat = Flatten()(conv_2)
hidden = Dense(self.pre_latent_size, activation='relu')(flat)
z_mean = Dense(self.latent_size)(hidden)
z_log_var = Dense(self.latent_size)(hidden)
# sample from normal with z_mean and z_log_var
z = Lambda(self.sampling, name='latent_space')([z_mean, z_log_var])
encoder_out_shape = tuple(z.get_shape().as_list())
# we instantiate these layers separately so as to reuse them later
input_decoder = Input(shape=encoder_out_shape[1:])
decoder_hid = Dense(self.pre_latent_size, activation='relu')
decoder_upsample = Dense(np.prod(conv2_out_shape[1:]),
activation='relu')
output_shape = (self.filters, conv2_out_shape[2], conv2_out_shape[3])
decoder_reshape = Reshape(output_shape)
decoder_deconv_1 = Conv2DTranspose(self.filters,
kernel_size=self.kernel_size,
padding='same',
activation='relu')
decoder_mean_squash = Conv2D(1,
kernel_size=self.kernel_size,
padding='same',
activation='sigmoid')
hid_decoded = decoder_hid(input_decoder)
up_decoded = decoder_upsample(hid_decoded)
reshape_decoded = decoder_reshape(up_decoded)
deconv_1_decoded = decoder_deconv_1(reshape_decoded)
x_decoded_mean = decoder_mean_squash(deconv_1_decoded)
# For parent fitting function
self.encoder = Model(input_encoder, z)
self.decoder = Model(input_decoder, x_decoded_mean)
self.model = Model(input_encoder,
self.decoder(self.encoder(input_encoder)))
# For parent loss function
self.z_mean = z_mean
self.z_log_var = z_log_var
self.z = z
def KerasModel(isCompile=True):
In = Input(shape=(None, 1, 1))
x = Lambda(fourierLayer, output_shape=fourierLayerShape)(In)
#x = Convolution2D(32, 7, 7, subsample=(3,3), activation='relu', W_regularizer=l2(L2_RATE))(x)
x = Conv2D(32, (7, 7),
activation='relu',
strides=(3, 3),
kernel_regularizer=l2(L2_RATE))(x)
#x=Conv2D(32, kernel_size=(7, 7), activation='relu')(x)
#x = Dropout(DROP_RATE)(x)
#x = Convolution2D(64, 7, 5, subsample=(3,3), activation='relu', W_regularizer=l2(L2_RATE))(x)
x = Conv2D(64, (7, 5),
activation='relu',
strides=(3, 3),
kernel_regularizer=l2(L2_RATE))(x)
#x=Conv2D(64, kernel_size=(7, 5), activation='relu')(x)
#x = Dropout(DROP_RATE)(x)
#x = Convolution2D(64, 3, 3, subsample=(2,2), activation='relu', W_regularizer=l2(L2_RATE))(x)
x = Conv2D(64, (2, 2),
activation='relu',
strides=(2, 2),
kernel_regularizer=l2(L2_RATE))(x)
#x=Conv2D(64, kernel_size=(3, 3), activation='relu')(x)
#x = Dropout(DROP_RATE)(x)
#x = Convolution2D(32, 3, 3, subsample=(2,2), activation='relu', W_regularizer=l2(L2_RATE))(x)
x = Conv2D(32, (2, 2),
activation='relu',
strides=(2, 2),
kernel_regularizer=l2(L2_RATE))(x)
# x=Conv2D(32, kernel_size=(3, 3), activation='relu')(x)
#x = Dropout(DROP_RATE)(x)
freq, chan = x.get_shape()[2:4]
x = TimeDistributed(Reshape([int(freq) * int(chan)]))(x)
x = Bidirectional(LSTM(128, ))(x)
#x = Dropout(DROP_RATE)(x)
x = Dense(64, activation='relu')(x)
#x = Dropout(DROP_RATE)(x)
x = Dense(24, activation='softmax')(x)
model = Model(inputs=In, outputs=x)
if isCompile:
opt = Adam(1e-4)
model.compile(
optimizer=opt,
loss='categorical_crossentropy',
metrics=['accuracy'],
)
return model
def f(x, chosen, style):
time_steps = int(x.get_shape()[1])
# Shift target one note to the left.
shift_chosen = Lambda(lambda x: tf.pad(
x[:, :, :-1, :], [[0, 0], [0, 0], [1, 0], [0, 0]]))(chosen)
# [batch, time, notes, 1]
total = int(shift_chosen.get_shape()[3]) * int(
shift_chosen.get_shape()[2]) * int(shift_chosen.get_shape()[1])
dim = int(total / (time_steps * NUM_NOTES))
shift_chosen_rsp = Reshape((time_steps, NUM_NOTES, dim))(shift_chosen)
##shift_chosen = Reshape((time_steps, NUM_NOTES, -1))(shift_chosen)
##shift_chosen_rsp=shift_chosen[:,:,:,-1]
##shift_chosen_rsp=Reshape((time_steps, NUM_NOTES, 1))(shift_chosen_rsp)
##shift_chosen_rsp = Reshape((time_steps, NUM_NOTES, 1))(shift_chosen)
# [batch, time, notes, features + 1]
x = Concatenate(axis=3)([x, shift_chosen_rsp])
for l in range(NOTE_AXIS_LAYERS):
# Integrate style
if l not in dense_layer_cache:
dense_layer_cache[l] = Dense(int(x.get_shape()[3]))
##a=x.get_shape()[3]
##dense_layer_cache[l] = Dense(int(a))
style_proj = dense_layer_cache[l](style)
style_proj = TimeDistributed(RepeatVector(NUM_NOTES))(style_proj)
style_proj = Activation('tanh')(style_proj)
style_proj = Dropout(dropout)(style_proj)
x = Add()([x, style_proj])
if l not in lstm_layer_cache:
lstm_layer_cache[l] = LSTM(NOTE_AXIS_UNITS,
return_sequences=True)
x = TimeDistributed(lstm_layer_cache[l])(x)
x = Dropout(dropout)(x)
return Concatenate()([note_dense(x), volume_dense(x)])
def set_model(self):
'''
Initialisers
'''
weight_seed = None
kernel_initializer = initializers.glorot_uniform(seed = weight_seed)
bias_initializer = initializers.glorot_uniform(seed = weight_seed)
'''
Encoder
'''
# define input with 'channels_first'
input_encoder = Input(shape=self.input_shape, name='encoder_input')
x = Conv2D(self.filters, self.kernel_size, strides=(2, 2), activation='relu', kernel_initializer=kernel_initializer, bias_initializer=bias_initializer, name='encoder_conv2D_1')(input_encoder)
x = Conv2D(2*self.filters, self.kernel_size, strides=(2, 2), activation='relu', kernel_initializer=kernel_initializer, bias_initializer=bias_initializer, name='encoder_conv2D_3')(x)
before_flatten_shape = tuple(x.get_shape().as_list())
x = Flatten()(x)
# x = Dense(self.pre_latent_size, activation='relu', name='encoder_dense_1')(x)
# separate dense layers for mu and log(sigma), both of size latent_dim
z_mean = Dense(self.latent_size, activation=None, kernel_initializer=kernel_initializer, bias_initializer=bias_initializer, name='encoder_z_mean')(x)
z_log_var = Dense(self.latent_size, activation=None, kernel_initializer=kernel_initializer, bias_initializer=bias_initializer, name='encoder_z_log_var')(x)
# sample from normal with z_mean and z_log_var
z = Lambda(self.sampling, name='encoder_z')([z_mean, z_log_var])
'''
Decoder
'''
# take encoder output shape
encoder_out_shape = tuple(z.get_shape().as_list())
# define rest of model
input_decoder = Input(shape=encoder_out_shape[1:], name='decoder_input')
# x = Dense(self.pre_latent_size, activation='relu')(input_decoder)
x = Dense(np.prod(before_flatten_shape[1:]), activation='relu')(input_decoder)
x = Reshape(before_flatten_shape[1:])(x)
x = Conv2DTranspose(self.filters, self.kernel_size, strides=(2, 2), activation='relu', kernel_initializer=kernel_initializer, bias_initializer=bias_initializer, name='encoder_conv2DT_2')(x)
decoded_img = Conv2DTranspose(1, self.kernel_size, strides=(2, 2), activation='sigmoid', kernel_initializer=kernel_initializer, bias_initializer=bias_initializer, name='encoder_conv2DT_3')(x)
'''
Necessary definitions
'''
# For parent fitting function
self.encoder = Model(input_encoder, z)
self.decoder = Model(input_decoder, decoded_img)
self.model = Model(input_encoder, self.decoder(self.encoder(input_encoder)))
# For parent loss function
self.z_mean = z_mean
self.z_log_var = z_log_var
self.z = z
def build_nn():
nn_input = Input((num_antennas, num_sub, 2))
dropout_rate = 0.25
num_complex_channels = 4
def k_mean(tensor):
return K.mean(tensor, axis=2)
mean_input = Lambda(k_mean)(nn_input)
print(mean_input.get_shape())
# complex to polar
real = Lambda(lambda x: x[:, :, :, 0])(nn_input)
imag = Lambda(lambda x: x[:, :, :, 1])(nn_input)
# complex_crop = Lambda(lambda x: x[:, :, 0, :], output_shape=(Nb_Antennas, 2, 1))(complex_input)
# complex_input = Reshape((Nb_Antennas, 2, 1))(mean_input)
real_squared = Multiply()([real, real])
imag_squared = Multiply()([imag, imag])
real_imag_squared_sum = Add()([real_squared, imag_squared])
# amplitude
def k_sqrt(tensor):
r = K.sqrt(tensor)
return r
r = Lambda(k_sqrt)(real_imag_squared_sum)
r = Reshape((num_antennas, num_sub, 1))(r)
print(r.get_shape())
# phase
def k_atan(tensor):
t = tf.math.atan2(tensor[0], tensor[1])
return t
t = Lambda(k_atan)([imag, real])
t = Reshape((num_antennas, num_sub, 1))(t)
print(t.get_shape())
polar_input = Concatenate()([r, t])
total_input = Concatenate()([nn_input, polar_input])
print("total", total_input.get_shape())
# reduce dimension of time axis
lay_input = Reshape((num_antennas, num_sub, num_complex_channels, 1))(total_input)
layD1 = Conv3D(8, (1, 23, num_complex_channels), strides=(1, 5, 1), padding='same')(lay_input)
layD1 = LeakyReLU(alpha=0.3)(layD1)
layD1 = Dropout(dropout_rate)(layD1)
layD2 = Conv3D(8, (1, 23, 1), padding='same')(layD1)
layD2 = LeakyReLU(alpha=0.3)(layD2)
layD2 = Concatenate()([layD1, layD2])
layD2 = Conv3D(8, (1, 1, num_complex_channels), padding='same')(layD2)
layD2 = LeakyReLU(alpha=0.3)(layD2)
layD2 = Conv3D(8, (1, 23, 1), strides=(1, 5, 1), padding='same',
kernel_regularizer=regularizers.l2(0.01))(layD2)
layD2 = LeakyReLU(alpha=0.3)(layD2)
layD2 = Dropout(dropout_rate)(layD2)
layD3 = Conv3D(8, (1, 23, 1), padding='same')(layD2)
layD3 = LeakyReLU(alpha=0.3)(layD3)
layD3 = Concatenate()([layD2, layD3])
layD3 = Conv3D(8, (1, 1, num_complex_channels), padding='same')(layD3)
layD3 = LeakyReLU(alpha=0.3)(layD3)
layD3 = Conv3D(8, (1, 23, 1), strides=(1, 5, 1), padding='same',
kernel_regularizer=regularizers.l2(0.01))(layD3)
layD3 = LeakyReLU(alpha=0.3)(layD3)
layD3 = Dropout(dropout_rate)(layD3)
layD4 = Conv3D(8, (1, 23, 1), padding='same')(layD3)
layD4 = LeakyReLU(alpha=0.3)(layD4)
layD4 = Concatenate()([layD4, layD3])
layD4 = Conv3D(8, (1, 1, num_complex_channels), padding='same')(layD4)
layD4 = LeakyReLU(alpha=0.3)(layD4)
# layD4 = Conv3D(8, (1, 23, 1), strides=(1, 5, 1), padding='same',
# kernel_regularizer=regularizers.l2(0.01))(layD4)
# layD4 = LeakyReLU(alpha=0.3)(layD4)
# layD4 = Dropout(dropout_rate)(layD4)
# conv over antenna layers
layV1 = Conv3D(8, (16, 1, 1), padding='same')(layD4)
layV1 = LeakyReLU(alpha=0.3)(layV1)
layV1 = Dropout(dropout_rate)(layV1)
layV1 = Concatenate()([layV1, layD4])
layV2 = Conv3D(8, (16, 1, 1), padding='same',
kernel_regularizer=regularizers.l2(0.01))(layV1)
layV2 = LeakyReLU(alpha=0.3)(layV2)
layV2 = Dropout(dropout_rate)(layV2)
layV2 = Concatenate()([layV2, layV1])
layV3 = Conv3D(8, (16, 1, 1), padding='same')(layV2)
layV3 = LeakyReLU(alpha=0.3)(layV3)
layV3 = Dropout(dropout_rate)(layV3)
# layV3 = Dropout(dropout_rate)(layV3)
# layV4 = Conv3D(8, (16, 1, 1), padding='valid')(layV3)
# layV4 = LeakyReLU(alpha=0.3)(layV4)
# layV4 = Dropout(dropout_rate)(layV4)
# layV5 = Conv3D(8, (16, 1, 1), padding='valid',
# kernel_regularizer=regularizers.l2(0.01))(layV4)
# layV5 = LeakyReLU(alpha=0.3)(layV5)
# layV5 = Dropout(dropout_rate)(layV5)
# layV6 = Conv3D(8, (16, 1, 1), padding='valid')(layV5)
# layV6 = LeakyReLU(alpha=0.3)(layV6)
# layV6 = Dropout(dropout_rate)(layV6)
# conv over complex layers
# layH1 = Conv3D(12, (1,1,2), strides=(1,1,2), padding='valid', activation='relu')(layV3)
# layH1 = Dropout(dropout_rate)(layH1)
# layH2 = Conv3D(12, (1,1,2), padding='same', activation='relu')(layH1)
# layH2 = Dropout(dropout_rate)(layH2)
# layH2 = Concatenate()([layH1, layH2])
# layH3 = Conv3D(12, (1,1,4), padding='same', activation='relu')(layH2)
# layH3 = Dropout(dropout_rate)(layH3)
nn_output = Flatten()(layV3)
nn_output = Dense(64, activation='relu')(nn_output)
nn_output = Dense(32, activation='relu')(nn_output)
nn_output = Dense(2, activation='linear')(nn_output)
nn = Model(inputs=nn_input, outputs=nn_output)
nn.compile(optimizer='Adam', loss='mse', metrics=[dist])
nn.summary()
return nn
bias_initializer=bias_initializer,
name='encoder_z_mean')(x)
z_log_var = Conv2D(4 * filters,
kernal_size,
activation=None,
kernel_initializer=kernel_initializer,
bias_initializer=bias_initializer,
name='encoder_z_log_var')(x)
# sample from normal with z_mean and z_log_var
z = Lambda(sampling, name='latent_space')([z_mean, z_log_var])
'''
Decoder
'''
# define input with 'channels_first'
encoder_out_shape = tuple(z.get_shape().as_list())
input_decoder = Input(shape=(encoder_out_shape[1], encoder_out_shape[2],
encoder_out_shape[3]),
name='decoder_input')
# transposed convolution and up sampling
x = Conv2DTranspose(2 * filters,
kernal_size,
activation='relu',
kernel_initializer=kernel_initializer,
bias_initializer=bias_initializer,
name='decoder_conv2DT_1')(input_decoder)
x = UpSampling2D(pool_size, name='decoder_up_sampling_1')(x)
# transposed convolution and up sampling
x = Conv2DTranspose(filters,
def channel(self, ind_emb):
#kernel_initializer = tf.contrib.layers.variance_scaling_initializer()
#bias_initializer = tf.constant_initializer(value=0.0)
ratio = 4
# with tf.variable_scope(name):
scale = []
output = []
channel = ind_emb[0].get_shape()[1]
shared_mean_1 = Dense(units=int(channel // ratio),
activation='relu',
kernel_initializer='random_uniform',
bias_initializer='zeros',
name='mean_1')
shared_mean_2 = Dense(units=int(channel),
kernel_initializer='random_uniform',
bias_initializer='zeros',
name='mean_2')
shared_max_1 = Dense(units=int(channel // ratio),
activation='relu',
kernel_initializer='random_uniform',
bias_initializer='zeros',
name='max_1')
shared_max_2 = Dense(units=int(channel),
kernel_initializer='random_uniform',
bias_initializer='zeros',
name='max_2')
shared_std_1 = Dense(units=int(channel // ratio),
activation='relu',
kernel_initializer='random_uniform',
bias_initializer='zeros',
name='std_1')
shared_std_2 = Dense(units=int(channel),
kernel_initializer='random_uniform',
bias_initializer='zeros',
name='std_2')
for i, emb in enumerate(ind_emb):
#emb = K.permute_dimensions(emb, (0, 2, 3,1))
avg_pool = Lambda(lambda x: K.mean(x, axis=[0, 2, 3]))(emb)
avg_pool = Lambda(lambda x: K.expand_dims(x, axis=0))(avg_pool)
assert avg_pool.get_shape()[-1] == channel
avg_pool = shared_mean_1(avg_pool)
assert avg_pool.get_shape()[-1] == channel // ratio
avg_pool = shared_mean_2(avg_pool)
assert avg_pool.get_shape()[-1] == channel
max_pool = Lambda(lambda x: K.max(x, axis=[0, 2, 3]))(emb)
max_pool = Lambda(lambda x: K.expand_dims(x, axis=0))(max_pool)
assert max_pool.get_shape()[-1] == channel
max_pool = shared_max_1(max_pool)
assert max_pool.get_shape()[-1] == channel // ratio
max_pool = shared_max_2(max_pool)
assert max_pool.get_shape()[-1] == channel
std_pool = Lambda(lambda x: K.std(x, axis=[0, 2, 3]))(emb)
std_pool = Lambda(lambda x: K.expand_dims(x, axis=0))(std_pool)
assert std_pool.get_shape()[-1] == channel
std_pool = shared_std_1(std_pool)
assert std_pool.get_shape()[-1] == channel // ratio
std_pool = shared_std_2(std_pool)
assert std_pool.get_shape()[-1] == channel
added = Add()([avg_pool, max_pool, std_pool])
activation = Activation('sigmoid')(added)
# scale = Lambda(lambda x: K.permute_dimensions(x,(1,0)))(activation)
# scale = Concatenate(axis=-1)([scale for j in range(channel)])
if i == 0:
# output = K.permute_dimensions(K.dot(K.permute_dimensions(emb, (0, 2, 3,1)),scale),(0,3,2,1))
emb = Lambda(lambda x: K.permute_dimensions(x, (0, 2, 3, 1)))(
emb)
mul = Multiply()([emb, activation])
output = Lambda(
lambda x: K.permute_dimensions(x, (0, 3, 1, 2)))(mul)
else:
# output = K.concatenate([output,K.permute_dimensions(K.dot(K.permute_dimensions(emb, (0, 2, 3,1)),scale),(0,3,2,1))],axis=1)
emb = Lambda(lambda x: K.permute_dimensions(x, (0, 2, 3, 1)))(
emb)
mul = Multiply()([emb, activation])
mul = Lambda(lambda x: K.permute_dimensions(x, (0, 3, 1, 2)))(
mul)
output = Concatenate(axis=1)([output, mul])
# emb = Lambda(lambda x: K.permute_dimensions(x,(0, 2, 3,1)))(ind_emb[0])
return output
def get_gru_ctc_model(
image_size=image_size,
seq_len=4, # 字符最大长
label_count=37): # 标签数量
img_height, img_width = image_size[0], image_size[1]
input_tensor = Input((img_height, img_width, 3), name='input')
x = input_tensor
x = preprocess(x)
x = Lambda(cnn_part)(x)
# x = cnn_part(x)
conv_shape = x.get_shape()
print(conv_shape)
x = Reshape(target_shape=(int(conv_shape[1]),
int(conv_shape[2] * conv_shape[3])))(x)
x = Dense(32, activation='relu')(x)
gru_1 = GRU(32,
return_sequences=True,
kernel_initializer='he_normal',
name='gru1')(x)
gru_1b = GRU(32,
return_sequences=True,
go_backwards=True,
kernel_initializer='he_normal',
name='gru1_b')(x)
gru1_merged = Add()([gru_1, gru_1b])
gru_2 = GRU(32,
return_sequences=True,
kernel_initializer='he_normal',
name='gru2')(gru1_merged)
gru_2b = GRU(32,
return_sequences=True,
go_backwards=True,
kernel_initializer='he_normal',
name='gru2_b')(gru1_merged)
x = Concatenate()([gru_2, gru_2b])
x = Dropout(0.25)(x)
x = Dense(label_count,
kernel_initializer='he_normal',
activation='softmax',
name='output')(x)
base_model = Model(inputs=input_tensor, outputs=x)
labels = Input(name='the_labels', shape=[seq_len], dtype='float32')
input_length = Input(name='input_length', shape=[1], dtype='int64')
label_length = Input(name='label_length', shape=[1], dtype='int64')
loss_out = Lambda(ctc_lambda_func, output_shape=(1, ),
name='ctc')([x, labels, input_length, label_length])
ctc_model = Model(
inputs=[input_tensor, labels, input_length, label_length],
outputs=[loss_out])
sgd = optimizers.Adam(lr=0.005)
ctc_model.compile(loss={
'ctc': lambda y_true, y_pred: y_pred
},
optimizer=sgd)
ctc_model.summary()
return conv_shape, base_model, ctc_model
def set_model(self):
'''
Initialisers
'''
weight_seed = None
kernel_initializer = initializers.glorot_uniform(seed=weight_seed)
bias_initializer = initializers.glorot_uniform(seed=weight_seed)
'''
Encoder
'''
# define input with 'channels_first'
input_encoder = Input(shape=input_shape, name='encoder_input')
# 'kernel_size' convolution with 'filters' output filters, stride 1x1 and 'valid' border_mode
x = Conv2D(self.filters,
self.kernel_size,
padding='valid',
activation='relu',
kernel_initializer=kernel_initializer,
bias_initializer=bias_initializer,
name='encoder_conv2D_1')(input_encoder)
x = MaxPooling2D(self.pool_size, name='encoder_max_pooling_1')(x)
x = Conv2D(2 * self.filters,
self.kernel_size,
padding='valid',
activation='relu',
kernel_initializer=kernel_initializer,
bias_initializer=bias_initializer,
name='encoder_conv2D_2')(x)
# separate dense layers for mu and log(sigma), both of size latent_dim
z_mean = Conv2D(self.latent_filters,
self.kernel_size,
padding='valid',
activation=None,
kernel_initializer=kernel_initializer,
bias_initializer=bias_initializer,
name='encoder_z_mean')(x)
z_log_var = Conv2D(self.latent_filters,
self.kernel_size,
padding='valid',
activation=None,
kernel_initializer=kernel_initializer,
bias_initializer=bias_initializer,
name='encoder_z_log_var')(x)
# sample from normal with z_mean and z_log_var
z = Lambda(self.sampling, name='latent_space')([z_mean, z_log_var])
'''
Decoder
'''
# define input with 'channels_first'
encoder_out_shape = tuple(z.get_shape().as_list())
input_decoder = Input(shape=(encoder_out_shape[1],
encoder_out_shape[2],
encoder_out_shape[3]),
name='decoder_input')
# transposed convolution and up sampling
x = Conv2DTranspose(2 * self.filters,
self.kernel_size,
padding='valid',
activation='relu',
kernel_initializer=kernel_initializer,
bias_initializer=bias_initializer,
name='decoder_conv2DT_1')(input_decoder)
x = Conv2DTranspose(self.filters,
self.kernel_size,
padding='valid',
activation='relu',
kernel_initializer=kernel_initializer,
bias_initializer=bias_initializer,
name='decoder_conv2DT_2')(x)
x = UpSampling2D(self.pool_size, name='decoder_up_sampling_1')(x)
x = Conv2DTranspose(1,
self.kernel_size,
padding='valid',
activation='sigmoid',
kernel_initializer=kernel_initializer,
bias_initializer=bias_initializer,
name='decoder_conv2DT_3')(x)
# define decoded image to be image in last layer
decoded_img = x
'''
Necessary definitions
'''
# For parent fitting function
self.encoder = Model(input_encoder, z)
self.decoder = Model(input_decoder, decoded_img)
self.model = Model(input_encoder,
self.decoder(self.encoder(input_encoder)))
# For parent loss function
self.z_mean = z_mean
self.z_log_var = z_log_var
self.z = z
def create_VAE(input_size=16):
'''
Returns a VAE model that takes in a image of size (input_size, input_size)
Inputs:
inputs_size = the size of input images to the encoder
Returns:
3-Tuple:
VAE model,
VAE encoder,
VAE decoder
'''
encoding_size = 3
epsilon_std = 1.0
conv_filters = 32
kernel_size = (3, 3)
pool_size = (2, 2)
conv_levels = 3
intermediate_size = 128
x = Input(shape=(input_size, input_size, 1))
h = Conv2D(conv_filters, kernel_size, activation='relu', padding='same')(x)
for _ in range(conv_levels - 1):
h = MaxPooling2D(pool_size, padding='same')(h)
h = Conv2D(conv_filters, kernel_size, activation='relu', padding='same')(h)
# h = MaxPooling2D(pool_size, padding='same')(h)
# h = Conv2D(cov_filters, kernel_size, activation='relu', padding='same')(h)
original_shape = h.get_shape().as_list()[1:]
h = Flatten()(h)
h = Dense(intermediate_size)(h)
z_mean = Dense(encoding_size)(h)
z_log_var = Dense(encoding_size)(h)
def sampling(args):
z_mean, z_log_sigma = args
epsilon = K.random_normal(shape=(K.shape(z_mean)[0], encoding_size), mean=0.,
stddev=epsilon_std)
return z_mean + K.exp(z_log_sigma / 2) * epsilon
z = Lambda(sampling)([z_mean, z_log_var])
generator_input = Input(shape=z.get_shape().as_list()[1:])
h = Dense(intermediate_size)
decode_h = h(z)
generator_h = h(generator_input)
h = Dense(np.cumprod(original_shape)[-1])
decode_h = h(decode_h)
generator_h = h(generator_h)
h = Reshape(original_shape)
decode_h = h(decode_h)
generator_h = h(generator_h)
h = Conv2D(conv_filters, kernel_size, activation='relu', padding='same')
decode_h = h(decode_h)
generator_h = h(generator_h)
for _ in range(conv_levels - 1):
h = UpSampling2D(pool_size)
decode_h = h(decode_h)
generator_h = h(generator_h)
h = Conv2D(conv_filters, kernel_size, activation='relu', padding='same')
decode_h = h(decode_h)
generator_h = h(generator_h)
h = Dense(1, activation='sigmoid')
decode_h = h(decode_h)
generator_h = h(generator_h)
encoder = Model(x, z)
generator = Model(generator_input, generator_h)
vae = Model(x, decode_h)
def vae_loss(x, x_decoded_mean):
xent_loss = K.sum(K.sum(metrics.binary_crossentropy(x, x_decoded_mean), axis=-1), axis=-1)
kl_loss = - 0.5 * K.mean(1 + z_log_var - K.square(z_mean) - K.exp(z_log_var), axis=-1)
return xent_loss + kl_loss
opt = optimizers.Adam()
vae.compile(optimizer=opt, loss=vae_loss)
generator.compile(optimizer=opt, loss='binary_crossentropy')
encoder.compile(optimizer=opt, loss='binary_crossentropy')
return vae, encoder, generator
return K.ctc_batch_cost(labels, y_pred, input_length, label_length)
rnn_size = 128
input_tensor = Input((width, height, 3))
x = input_tensor
x = Lambda(lambda x: (x - 127.5) / 127.5)(x)
for i in range(3):
for j in range(2):
x = Conv2D(32 * 2**i, 3, kernel_initializer='he_uniform')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = MaxPooling2D(pool_size=(2, 2))(x)
conv_shape = x.get_shape().as_list()
rnn_length = conv_shape[1]
rnn_dimen = conv_shape[2] * conv_shape[3]
print(conv_shape, rnn_length, rnn_dimen)
x = Reshape(target_shape=(rnn_length, rnn_dimen))(x)
rnn_length -= 2
x = Dense(rnn_size, kernel_initializer='he_uniform')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Dropout(0.2)(x)
gru_1 = GRU(rnn_size,
return_sequences=True,
kernel_initializer='he_uniform',
def gan_2(input_img,
input_channel=3,
z_num=64,
no_of_pairs=5,
hidden_num=128,
activation_fn=tf.nn.elu,
noise_dim=64):
# Encoder
encoder_layer_list = []
x = Conv2D(hidden_num,
kernel_size=3,
strides=1,
activation=activation_fn,
padding='same')(input_img)
prev_channel_num = hidden_num
for idx in range(no_of_pairs):
# to increase number of filter by (filters)*(index+1) ex: 16, 32, 48 ...
channel_num = hidden_num * (idx + 1)
res = x
x = Conv2D(channel_num,
kernel_size=3,
strides=1,
activation=activation_fn,
padding='same')(x)
x = Conv2D(channel_num,
kernel_size=3,
strides=1,
activation=activation_fn,
padding='same')(x)
if idx > 0:
encoder_layer_list.append(x)
if idx < no_of_pairs - 1:
x = Conv2D(channel_num,
kernel_size=3,
strides=2,
activation=activation_fn,
padding='same')(x)
if noise_dim > 0:
x = Lambda(get_noise)(x)
for idx in range(no_of_pairs):
if idx < no_of_pairs - 1:
x = Concatenate(axis=-1)(
[x, encoder_layer_list[no_of_pairs - 2 - idx]])
channel_num = x.get_shape().as_list()[-1]
x = Conv2D(hidden_num,
kernel_size=3,
strides=1,
activation=activation_fn,
padding='same')(x)
x = Conv2D(hidden_num,
kernel_size=3,
strides=1,
activation=activation_fn,
padding='same')(x)
if idx < no_of_pairs - 1:
x = UpSampling2D(2)(x)
out = Conv2D(input_channel,
kernel_size=3,
strides=1,
activation=None,
padding='same')(x)
return out
def googleNet_n(x,
data_format='channels_last',
num_classes=24,
num_layers=[1, 1, 2, 1],
features=[1, 1, 1, 1, 1]):
xft = Lambda(lambda v: tf_fft(v))(x)
x = Reshape(in_shp + (1, ), input_shape=in_shp)(x)
x = Conv2D(filters=64 * features[0],
kernel_size=[2, 7],
strides=[2, 2],
data_format=data_format,
padding='same',
activation='relu')(x)
x = MaxPooling2D([1, 3], strides=[1, 2], padding='same')(x)
for dep in range(num_layers[0]):
x = Conv2D(filters=192 * features[1],
kernel_size=[1, 3],
strides=[1, 1],
padding='same',
activation='relu')(x)
x = MaxPooling2D([1, 3], strides=[1, 2], padding='same')(x)
xft = Reshape(in_shp + (1, ), input_shape=in_shp)(xft)
xft = Conv2D(filters=64 * features[0],
kernel_size=[2, 4],
strides=[2, 2],
data_format=data_format,
padding='same',
activation='relu')(xft)
xft = MaxPooling2D([1, 3], strides=[1, 2], padding='same')(xft)
for dep in range(num_layers[0]):
xft = Conv2D(filters=192 * features[1],
kernel_size=[1, 3],
strides=[1, 1],
padding='same',
activation='relu')(xft)
xft = MaxPooling2D([1, 3], strides=[1, 2], padding='same')(xft)
print(x.get_shape(), xft.get_shape())
x = keras.layers.concatenate([x, xft], axis=3)
print(x.get_shape())
for dep in range(num_layers[1]):
x = inception(x,
height=2,
fs=np.array([32, 32, 32, 32, 32]) * features[2],
tw_tower=True)
x = MaxPooling2D([1, 3], strides=2, padding='same')(x)
for dep in range(num_layers[2]):
x = inception(x,
height=2,
fs=np.array([48, 96, 48, 96, 96]) * features[3],
with_residual=True)
#out_mid = out_tower(x, dr=0.3)
#for dep in range(num_layers[3]):
# x = inception(x, height=2, fs=np.array([48,96,48,96,96])*features[4], with_residual=True)
x = MaxPooling2D([2, 3], strides=2, padding='same')(x)
for dep in range(num_layers[3]):
x = inception(x,
height=1,
fs=np.array([32, 32, 32, 32, 32]) * features[4])
out = out_tower(x, dr=0.5, reg=args.confireg)
#out = Average()([out_mid, out_late])
return out
def prot_encoder(model, seq, seq_size):
encoder = model.prot_encoder
if model.dataset.name in LIST_AA_DATASETS:
if encoder['conv_strides'] != 1 or encoder['name'] == 'conv_biLSTM_att'\
or model.batch_size != 1:
print('!! ERROR !!: AA prediction but conv_strides > 1 ' +
'OR tt_mech at the end of encoder OR batch_size > 1 !!')
exit(1)
kreg, breg = None, None
# if model.prot_reg != 0:
# kreg, breg = regularizers.l2(model.prot_reg), regularizers.l2(model.prot_reg)
# else:
# kreg, breg = None, None
seq = Lambda(lambda t: expand_last_dim(t), name="expand_last_dim")(seq)
if encoder['name'] == 'conv' or encoder['name'] == 'conv_aa':
n_steps = encoder['n_steps']
n_filters = encoder['nb_conv_filters']
# seq = Lambda(lambda t: t, name='temp1')(seq)
for n in range(n_steps):
seq_shape = seq.get_shape().as_list()
kernel_size = (seq_shape[1], encoder['filter_size'])
strides = (seq_shape[1], encoder['conv_strides'])
seq = seq_conv(model, seq, n_filters, kernel_size, strides, kreg,
breg, model.prot_reg, model.prot_dropout,
model.prot_BN, n)
print('seq', seq)
seq_size = Lambda(lambda t: update_seq_size(t, float(strides[1])))(
seq_size)
# seq = Lambda(lambda t: t, name='temp2')(seq)
if model.dataset.name not in LIST_AA_DATASETS and 'aa' not in encoder[
'name']:
embedding = Lambda(lambda t: agg_sum(t, 2), name='agg_sum')(seq)
print('###', embedding)
embedding = Lambda(lambda t: squeeze(t, -1),
name='prot_embedding')(embedding)
elif 'aa' in encoder['name']:
seq = Lambda(lambda t: squeeze(t, -1), name='conv_emb')(seq)
seq = Permute((2, 1))(seq)
embedding = Lambda(lambda t: t, name='prot_embedding')(seq)
else:
seq = Permute((2, 1, 3))(seq)
embedding = Lambda(lambda t: squeeze(squeeze(t, -1), 0),
name='prot_embedding')(seq)
elif 'conv_biLSTM' in encoder['name'] or 'conv_biGRU' in encoder['name']:
seq_shape = seq.get_shape().as_list()
n_filters = encoder['nb_conv_filters']
kernel_size = (seq_shape[1], encoder['filter_size'])
strides = (seq_shape[1], encoder['conv_strides'])
seq_size = Lambda(lambda t: update_seq_size(t, float(strides[1])))(
seq_size)
seq = seq_conv(model, seq, n_filters, kernel_size, strides, kreg, breg,
model.prot_reg, model.prot_dropout, model.prot_BN, 0)
seq = Lambda(lambda t: squeeze(t, -1), name='conv_emb')(seq)
seq = Permute((2, 1))(seq)
if model.dataset.name not in LIST_AA_DATASETS:
seq = Masking(mask_value=0.0)(seq)
rseq = True if 'att' not in encoder['name'] and 'aa' not in encoder[
'name'] else True
if 'biLSTM' in encoder['name']:
lstm_layer = LSTM(n_filters,
return_sequences=rseq,
activation='tanh',
kernel_regularizer=kreg,
recurrent_regularizer=kreg,
bias_regularizer=breg,
dropout=model.prot_dropout)
elif 'biGRU' in encoder['name']:
lstm_layer = GRU(n_filters,
return_sequences=rseq,
activation='tanh',
kernel_regularizer=kreg,
recurrent_regularizer=kreg,
bias_regularizer=breg,
dropout=model.prot_dropout)
# import pdb; pdb.Pdb().set_trace()
seq = Bidirectional(lstm_layer,
merge_mode='concat',
weights=None,
name='biLSTM')(seq)
if model.prot_reg != 0:
seq = GaussianNoise(model.prot_reg)(seq)
if 'att' in encoder['name']:
attention_probs = Dense(n_filters * 2,
activation='softmax',
name='attention_probs')(seq)
embedding = Multiply(name='prot_embedding')([seq, attention_probs])
elif model.dataset.name in LIST_AA_DATASETS or 'aa' in encoder['name']:
embedding = Lambda(lambda t: t, name='prot_embedding')(seq)
else:
embedding = Lambda(lambda t: agg_sum(t, 1),
name='prot_embedding')(seq)
# embedding = Lambda(lambda t: t, name='prot_embedding')(seq)
# seq = Permute((2, 1))(seq)
# import pdb; pdb.Pdb().set_trace()
# embedding = Lambda(lambda t: squeeze(t, 0), name='prot_embedding')(seq)
if encoder['hand_crafted_features']:
embedding = Concatenate(axis=-1)([embedding, model.features])
# import pdb; pdb.Pdb().set_trace()
return embedding, seq_size
def build_ctc_network(args, training=True):
global N_COL
global N_ROW
"""
if args.stn == True:
from STN.STN import locnet
input_tensor = locnet(input, sampling_size = (N_COL, N_ROW))
print("STN")
"""
#stn_model = None
if hasattr(args, 'stn_model'):
pass
else:
args.stn_model = False
if args.model == 'resnet18':
NOT_CARE = 1
base_model = resnet.ResnetBuilder.build_resnet_18(input_shape=(N_COL,
N_ROW,
3),
num_outputs=NOT_CARE,
include_top=False,
args=args)
elif args.model == 'resnet18_2222':
NOT_CARE = 1
base_model = resnet.ResnetBuilder.build_resnet_18_2222(
input_shape=(N_COL, N_ROW, 3),
num_outputs=NOT_CARE,
include_top=False,
args=args)
elif args.model == 'resnet18_2222_64':
NOT_CARE = 1
base_model = resnet.ResnetBuilder.build_resnet_18_2222_start_from64(
input_shape=(N_COL, N_ROW, 3),
num_outputs=NOT_CARE,
include_top=False,
args=args)
elif args.model == 'resnet18_2222_48':
NOT_CARE = 1
base_model = resnet.ResnetBuilder.build_resnet_18_2222_start_from48(
input_shape=(N_COL, N_ROW, 3),
num_outputs=NOT_CARE,
include_top=False,
args=args)
elif args.model == 'resnet18_2222_32':
NOT_CARE = 1
base_model = resnet.ResnetBuilder.build_resnet_18_2222_start_from32(
input_shape=(N_COL, N_ROW, 3),
num_outputs=NOT_CARE,
include_top=False,
args=args)
elif args.model == 'resnet18_2222_16':
NOT_CARE = 1
base_model = resnet.ResnetBuilder.build_resnet_18_2222_start_from16(
input_shape=(N_COL, N_ROW, 3),
num_outputs=NOT_CARE,
include_top=False,
args=args)
elif args.model == 'resnet34':
NOT_CARE = 1
base_model = resnet.ResnetBuilder.build_resnet_34(input_shape=(N_COL,
N_ROW,
3),
num_outputs=NOT_CARE,
include_top=False,
args=args)
elif args.model == 'resnet50':
NOT_CARE = 1
base_model = resnet.ResnetBuilder.build_resnet_50(input_shape=(N_COL,
N_ROW,
3),
num_outputs=NOT_CARE,
include_top=False,
args=args)
else:
raise TypeError('model should be in the list of the supported model!')
#print('Input col: ', N_COL)
#print('Input row: ', N_ROW)
x = base_model.output
#CNN to RNN
x = Lambda(lambda x: K.permute_dimensions(x, (0, 2, 1, 3)))(
x) # switchaxes from [b,h,w,c] to [b,w,h,c]
conv_shape = x.get_shape() # b, h,w,c resnet 18 -> (?, 8, 16, 256)
#print('conv_shape', conv_shape)
x = Reshape(target_shape=(int(conv_shape[1]),
int(conv_shape[2] * conv_shape[3])),
name='reshape')(x) #(?, 16, 8, 256)
x = Dense(para.dense_size,
activation='relu',
kernel_initializer='he_normal',
name='dense1')(x) #5_exp ->128
#x = BatchNormalization()(x)
# GRU RNN
GRU_UNIT = CuDNNGRU
bi_gru1 = Bidirectional(GRU_UNIT(para.rnn_size,
return_sequences=True,
kernel_initializer='he_normal',
name='bi_gru1'),
merge_mode='sum')(x)
bi_gru1 = BatchNormalization()(bi_gru1)
bi_gru2 = Bidirectional(GRU_UNIT(para.rnn_size,
return_sequences=True,
kernel_initializer='he_normal',
name='bi_gru2'),
merge_mode='concat')(bi_gru1)
bi_gru2 = BatchNormalization()(bi_gru2)
#attention
#att = AttentionWithContext()(bi_gru2)
inner = Dense(para.num_classes,
kernel_initializer='he_normal',
name='dense2')(bi_gru2)
y_pred = Activation('softmax', name='softmax')(inner)
labels = Input(name='the_labels',
shape=[para.max_text_len],
dtype='float32') # (None ,7)
input_length = Input(name='input_length', shape=[1],
dtype='int64') # (None, 1)
label_length = Input(name='label_length', shape=[1],
dtype='int64') # (None, 1)
# Keras doesn't currently support loss funcs with extra parameters
# so CTC loss is implemented in a lambda layer
loss_out = Lambda(ctc_lambda_func, output_shape=(1, ),
name='ctc')([y_pred, labels, input_length,
label_length]) #(None, 1)
if training:
return Model(
inputs=[base_model.input, labels, input_length, label_length],
outputs=loss_out), conv_shape[1]
else:
return Model(inputs=[base_model.input], outputs=y_pred)
def build_model(width: int, height: int, rnn_size: int, classify_count: int,
classify_len: int, learning_rate: float):
input_tensor = Input((width, height, 3))
x = input_tensor
x = Lambda(lambda a: (a - 127.5) / 127.5)(x)
for j in range(3):
for i in range(2):
x = Conv2D(32 * 2**i, (3, 3), kernel_initializer='he_uniform')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = MaxPooling2D((2, 2))(x)
conv_shape = x.get_shape().as_list()
rnn_length = conv_shape[1]
rnn_dimen = conv_shape[2] * conv_shape[3]
x = Reshape((rnn_length, rnn_dimen))(x)
rnn_length -= 2
x = Dense(rnn_size, kernel_initializer='he_uniform')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Dropout(0.2)(x)
gru_1 = GRU(rnn_size,
return_sequences=True,
kernel_initializer='he_uniform',
name='gru1')(x)
gru_1b = GRU(rnn_size,
return_sequences=True,
kernel_initializer='he_uniform',
go_backwards=True,
name='gru1_b')(x)
x = add([gru_1, gru_1b])
gru_2 = GRU(rnn_size,
return_sequences=True,
kernel_initializer='he_uniform',
name='gru2')(x)
gru_2b = GRU(rnn_size,
return_sequences=True,
kernel_initializer='he_uniform',
go_backwards=True,
name='gru2_b')(x)
x = concatenate([gru_2, gru_2b])
x = Dropout(0.2)(x)
x = Dense(classify_count, activation='softmax')(x)
base_model = Model(inputs=input_tensor, outputs=x)
labels = Input(name='the_labels', shape=[classify_len], dtype='float32')
input_length = Input(name='input_length', shape=[1], dtype='int64')
label_length = Input(name='label_length', shape=[1], dtype='int64')
loss_out = Lambda(ctc_lambda, output_shape=(1, ),
name='ctc')([x, labels, input_length, label_length])
opt = Adam(lr=learning_rate)
ctc_model = Model(
inputs=[input_tensor, labels, input_length, label_length],
outputs=[loss_out])
ctc_model.compile(loss={
'ctc': lambda y_true, y_pred: y_pred
},
optimizer=opt)
return base_model, ctc_model, rnn_length
def NN_model(args, training=True):
global N_COL
global N_ROW
if args.model == 'densenet121':
from keras.applications.densenet import DenseNet121
input_tensor = Input(shape=(N_COL, N_ROW, 3))
base_model = DenseNet121(input_shape=(N_COL, N_ROW, 3),
include_top=False,
weights='imagenet',
input_tensor=input_tensor,
pooling=None)
elif args.model == 'resnet18':
import resnet
NOT_CARE = 1
base_model = resnet.ResnetBuilder.build_resnet_18(input_shape=(N_COL,
N_ROW,
3),
num_outputs=NOT_CARE,
include_top=False)
elif args.model == 'resnet18_2222':
import resnet
NOT_CARE = 1
base_model = resnet.ResnetBuilder.build_resnet_18_2222(
input_shape=(N_COL, N_ROW, 3),
num_outputs=NOT_CARE,
include_top=False)
elif args.model == 'resnet34':
import resnet
NOT_CARE = 1
base_model = resnet.ResnetBuilder.build_resnet_34(input_shape=(N_COL,
N_ROW,
3),
num_outputs=NOT_CARE,
include_top=False)
elif args.model == 'resnet50':
import resnet
NOT_CARE = 1
base_model = resnet.ResnetBuilder.build_resnet_50(input_shape=(N_COL,
N_ROW,
3),
num_outputs=NOT_CARE,
include_top=False)
elif args.model == 'resnet101':
import resnet
NOT_CARE = 1
base_model = resnet.ResnetBuilder.build_resnet_101(
input_shape=(N_COL, N_ROW, 3),
num_outputs=NOT_CARE,
include_top=False)
else:
raise TypeError('model should be in the list of the supported model!')
print('Input col: ', N_COL)
print('Input row: ', N_ROW)
x = base_model.output
#CNN to RNN
x = Lambda(lambda x: K.permute_dimensions(x, (0, 2, 1, 3)))(
x) # switchaxes from [b,h,w,c] to [b,w,h,c]
conv_shape = x.get_shape() # b, h,w,c resnet 18 -> (?, 16, 32, 256)
print('conv_shape', conv_shape)
x = Reshape(target_shape=(int(conv_shape[1]),
int(conv_shape[2] * conv_shape[3])),
name='reshape')(x)
x = Dense(para.dense_size,
activation='relu',
kernel_initializer='he_normal',
name='dense1')(x)
#x = BatchNormalization()(x)
# GRU RNN
gru_1 = GRU(para.rnn_size,
return_sequences=True,
init='he_normal',
name='gru1')(x)
gru_1b = GRU(para.rnn_size,
return_sequences=True,
go_backwards=True,
init='he_normal',
name='gru1_b')(x)
gru1_merged = add([gru_1, gru_1b])
gru1_merged = BatchNormalization()(gru1_merged)
gru_2 = GRU(para.rnn_size,
return_sequences=True,
init='he_normal',
name='gru2')(gru1_merged)
gru_2b = GRU(para.rnn_size,
return_sequences=True,
go_backwards=True,
init='he_normal',
name='gru2_b')(gru1_merged)
gru2_merged = concatenate([gru_2, gru_2b])
gru2_merged = BatchNormalization()(gru2_merged)
inner = Dense(para.num_classes,
kernel_initializer='he_normal',
name='dense2')(gru2_merged)
y_pred = Activation('softmax', name='softmax')(inner)
labels = Input(name='the_labels',
shape=[para.max_text_len],
dtype='float32') # (None ,7)
input_length = Input(name='input_length', shape=[1],
dtype='int64') # (None, 1)
label_length = Input(name='label_length', shape=[1],
dtype='int64') # (None, 1)
# Keras doesn't currently support loss funcs with extra parameters
# so CTC loss is implemented in a lambda layer
loss_out = Lambda(ctc_lambda_func, output_shape=(1, ),
name='ctc')([y_pred, labels, input_length,
label_length]) #(None, 1)
if training:
return Model(
inputs=[base_model.input, labels, input_length, label_length],
outputs=loss_out), conv_shape[1]
else:
return Model(inputs=[base_model.input], outputs=y_pred)
def build_nn(num_antenna=64):
nn_input = Input((num_antenna, num_sub, 2))
dropout_rate = 0.25
num_complex_channels = 6
def k_mean(tensor):
return K.mean(tensor, axis=2)
mean_input = Lambda(k_mean)(nn_input)
print(mean_input.get_shape())
# complex to polar
real = Lambda(lambda x: x[:, :, :, 0])(nn_input)
imag = Lambda(lambda x: x[:, :, :, 1])(nn_input)
# complex_crop = Lambda(lambda x: x[:, :, 0, :], output_shape=(Nb_Antennas, 2, 1))(complex_input)
# complex_input = Reshape((Nb_Antennas, 2, 1))(mean_input)
real_squared = Multiply()([real, real])
imag_squared = Multiply()([imag, imag])
real_imag_squared_sum = Add()([real_squared, imag_squared])
# amplitude
def k_sqrt(tensor):
r = K.sqrt(tensor)
return r
r = Lambda(k_sqrt)(real_imag_squared_sum)
r = Reshape((num_antenna, num_sub, 1))(r)
print(r.get_shape())
# phase
def k_atan(tensor):
import tensorflow as tf
t = tf.math.atan2(tensor[0], tensor[1])
return t
t = Lambda(k_atan)([imag, real])
t = Reshape((num_antenna, num_sub, 1))(t)
print(t.get_shape())
def ifft(x):
y = tf.complex(x[:, :, :, 0], x[:, :, :, 1])
ifft = tf.spectral.ifft(y)
return tf.stack([tf.math.real(ifft), tf.math.imag(ifft)], axis=3)
polar_input = Concatenate()([r, t])
time_input = Lambda(ifft)(nn_input)
total_input = Concatenate()([nn_input, polar_input, time_input])
# print("total", total_input.get_shape())
# reduce dimension of time axis
lay_input = Reshape(
(num_antenna, num_sub, num_complex_channels, 1))(total_input)
layD1 = Conv3D(8, (1, 23, num_complex_channels),
strides=(1, 5, 1),
padding='same')(lay_input)
layD1 = LeakyReLU(alpha=0.3)(layD1)
layD1 = Dropout(dropout_rate)(layD1)
layD2 = Conv3D(8, (1, 23, 1), padding='same')(layD1)
layD2 = LeakyReLU(alpha=0.3)(layD2)
layD2 = Concatenate()([layD1, layD2])
layD2 = Conv3D(8, (1, 1, num_complex_channels), padding='same')(layD2)
layD2 = LeakyReLU(alpha=0.3)(layD2)
layD2 = Conv3D(8, (1, 23, 1),
strides=(1, 5, 1),
padding='same',
kernel_regularizer=regularizers.l2(0.01))(layD2)
layD2 = LeakyReLU(alpha=0.3)(layD2)
layD2 = Dropout(dropout_rate)(layD2)
layD3 = Conv3D(8, (1, 23, 1), padding='same')(layD2)
layD3 = LeakyReLU(alpha=0.3)(layD3)
layD3 = Concatenate()([layD2, layD3])
layD3 = Conv3D(8, (1, 1, num_complex_channels), padding='same')(layD3)
layD3 = LeakyReLU(alpha=0.3)(layD3)
layD3 = Conv3D(8, (1, 23, 1),
strides=(1, 5, 1),
padding='same',
kernel_regularizer=regularizers.l2(0.01))(layD3)
layD3 = LeakyReLU(alpha=0.3)(layD3)
layD3 = Dropout(dropout_rate)(layD3)
layD4 = Conv3D(8, (1, 23, 1), padding='same')(layD3)
layD4 = LeakyReLU(alpha=0.3)(layD4)
layD4 = Concatenate()([layD4, layD3])
layD4 = Conv3D(8, (1, 1, num_complex_channels), padding='same')(layD4)
layD4 = LeakyReLU(alpha=0.3)(layD4)
layV1 = Conv3D(8, (8, 1, 1), padding='same')(layD4)
layV1 = LeakyReLU(alpha=0.3)(layV1)
layV1 = Dropout(dropout_rate)(layV1)
layV1 = Concatenate()([layV1, layD4])
layV2 = Conv3D(8, (8, 1, 1),
padding='same',
kernel_regularizer=regularizers.l2(0.01))(layV1)
layV2 = LeakyReLU(alpha=0.3)(layV2)
layV2 = Dropout(dropout_rate)(layV2)
layV2 = Concatenate()([layV2, layV1])
layV3 = Conv3D(8, (8, 1, 1), padding='same')(layV2)
layV3 = LeakyReLU(alpha=0.3)(layV3)
layV3 = Dropout(dropout_rate)(layV3)
layV3 = Concatenate()([layV3, layV2])
layV4 = Conv3D(8, (8, 1, 1), padding='same')(layV3)
layV4 = LeakyReLU(alpha=0.3)(layV4)
layV4 = Dropout(dropout_rate)(layV4)
layV4 = Concatenate()([layV4, layV3])
layV5 = Conv3D(8, (8, 1, 1), padding='same')(layV4)
layV5 = LeakyReLU(alpha=0.3)(layV5)
layV5 = Dropout(dropout_rate)(layV5)
nn_output = Flatten()(layV5)
nn_output = Dense(64, activation='relu')(nn_output)
nn_output = Dense(32, activation='relu')(nn_output)
nn_output = Dense(2, activation='linear')(nn_output)
nn = Model(inputs=nn_input, outputs=nn_output)
# nn = multi_gpu_model(nn, gpus=2)
nn.compile(optimizer='Adam', loss='mse', metrics=[dist])
nn.summary()
return nn
def build_model(train_batch_size, input_shape, smpl_path, output_wh, num_classes,
encoder_architecture='resnet50', use_IEF=False, vertex_sampling=None,
scaledown=0.005):
"""
Function to build indirect learning via body-part segmentation model. Builds encoder,
decoder and segmenter.
:param train_batch_size: training batch size.
:param input_shape: (H, W, 3)
:param smpl_path: path to SMPL model parameters.
:param output_wh: output width and height
:param num_classes: number of body-part classes
:param encoder_architecture: resnet50 or enet
:param use_IEF: use iterative error feedback flag
:param vertex_sampling:
:param scaledown: how much to scaledown regression module output by
:return: segs_model, smpl_model, verts_model, projects_model
"""
num_camera_params = 4
num_smpl_params = 72 + 10
num_total_params = num_smpl_params + num_camera_params
# --- BACKBONE ---
if encoder_architecture == 'enet':
inp = Input(shape=input_shape)
img_features = build_enet(inp) # (N, 32, 32, 128) output size from enet
img_features = Conv2D(256, (3, 3), activation='relu', padding='same')(img_features)
img_features = BatchNormalization()(img_features)
img_features = MaxPooling2D(pool_size=(4, 4))(img_features)
img_features = Conv2D(512, (3, 3), activation='relu', padding='same')(img_features)
img_features = BatchNormalization()(img_features)
img_features = MaxPooling2D()(img_features)
img_features = Conv2D(2048, (3, 3), activation='relu', padding='same')(img_features)
img_features = BatchNormalization()(img_features)
img_features = MaxPooling2D(pool_size=(4, 4))(img_features)
img_features = Reshape((2048,))(img_features)
elif encoder_architecture == 'resnet50':
resnet = resnet50.ResNet50(include_top=False, weights=None, input_shape=input_shape)
inp = resnet.input
img_features = resnet.output
img_features = Reshape((2048,))(img_features)
# --- REGRESSION MODULE ---
if use_IEF:
# --- IEF MODULE ---
# Instantiate ief layers
IEF_layer_1 = Dense(1024, activation='relu', name='IEF_layer_1')
IEF_layer_2 = Dense(1024, activation='relu', name='IEF_layer_2')
IEF_layer_3 = Dense(num_total_params, activation='linear', name='IEF_layer_3')
state1 = Lambda(concat_mean_param,
arguments={'img_wh': output_wh})(img_features)
param1 = Lambda(lambda x: x[:, 2048:])(state1)
print("State1 shape", state1.get_shape())
print("Param1 shape", param1.get_shape())
# Iteration 1
delta1 = IEF_layer_1(state1)
delta1 = IEF_layer_2(delta1)
delta1 = IEF_layer_3(delta1)
delta1 = Lambda(lambda x, d: x * d, arguments={"d": scaledown})(delta1)
param2 = Add()([param1, delta1])
state2 = Concatenate()([img_features, param2])
# Iteration 2
delta2 = IEF_layer_1(state2)
delta2 = IEF_layer_2(delta2)
delta2 = IEF_layer_3(delta2)
delta2 = Lambda(lambda x, d: x * d, arguments={"d": scaledown})(delta2)
param3 = Add()([param2, delta2])
state3 = Concatenate()([img_features, param3])
# Iteration 3
delta3 = IEF_layer_1(state3)
delta3 = IEF_layer_2(delta3)
delta3 = IEF_layer_3(delta3)
delta3 = Lambda(lambda x, d: x * d, arguments={"d": scaledown})(delta3)
final_param = Add()([param3, delta3])
else:
smpl = Dense(2048, activation='relu')(img_features)
smpl = Dense(1024, activation='relu')(smpl)
smpl = Dense(num_total_params, activation='linear')(smpl)
smpl = Lambda(lambda x: x * scaledown, name="scale_down")(smpl)
final_param = Lambda(load_mean_set_cam_params,
arguments={'img_wh': output_wh})(smpl)
# --- DECODER ---
verts = SMPLLayer(smpl_path, batch_size=train_batch_size)(final_param)
# --- SEGMENTER ---
projects_with_depth = Lambda(orthographic_project,
arguments={'vertex_sampling': vertex_sampling},
name='project')([verts, final_param])
masks = Lambda(compute_mask, name='compute_mask')(projects_with_depth)
segs = Lambda(projects_to_seg,
arguments={'img_wh': output_wh,
'vertex_sampling': vertex_sampling},
name='segment')([projects_with_depth, masks])
segs = Reshape((output_wh * output_wh, num_classes), name="final_reshape")(segs)
segs = Activation('softmax', name="final_softmax")(segs)
segs_model = Model(inputs=inp, outputs=segs)
smpl_model = Model(inputs=inp, outputs=final_param)
verts_model = Model(inputs=inp, outputs=verts)
projects_model = Model(inputs=inp, outputs=projects_with_depth)
print(segs_model.summary())
return segs_model, smpl_model, verts_model, projects_model
def create_model(existing='',
is_twohundred=False,
is_halffeatures=True,
weights='imagenet'):
if len(existing) == 0:
print('Loading base model (DenseNet)..')
input = Input(shape=(None, None, None))
image, aux_output, has_locations = Lambda(split_input)(input)
# Encoder Layers
if is_twohundred:
base_model = applications.DenseNet201(input_tensor=image,
weights=weights,
include_top=False)
else:
base_model = applications.DenseNet169(input_tensor=image,
weights=weights,
include_top=False)
# Starting point for decoder
base_model_output_shape = base_model.layers[-1].output.shape
print('Base model loaded [output: %s].' % (base_model_output_shape, ))
# Layer freezing?
for layer in base_model.layers:
layer.trainable = True
# Starting number of decoder filters
if is_halffeatures:
decode_filters = int(int(base_model_output_shape[-1]) / 2)
else:
decode_filters = int(base_model_output_shape[-1])
# Define upsampling layer
def upproject(tensor, filters, name, concat_with):
up_i = BilinearUpSampling2D((2, 2),
name=name + '_upsampling2d')(tensor)
up_i = Concatenate(name=name + '_concat')(
[up_i,
base_model.get_layer(concat_with).output]) # Skip connection
up_i = Conv2D(filters=filters,
kernel_size=3,
strides=1,
padding='same',
name=name + '_convA')(up_i)
up_i = LeakyReLU(alpha=0.2)(up_i)
up_i = Conv2D(filters=filters,
kernel_size=3,
strides=1,
padding='same',
name=name + '_convB')(up_i)
up_i = LeakyReLU(alpha=0.2)(up_i)
return up_i
# Decoder Layers
decoder = Conv2D(filters=decode_filters,
kernel_size=1,
padding='same',
input_shape=base_model_output_shape,
name='conv2')(base_model.output)
decoder = upproject(decoder,
int(decode_filters / 2),
'up1',
concat_with='pool3_pool')
decoder = upproject(decoder,
int(decode_filters / 4),
'up2',
concat_with='pool2_pool')
decoder = upproject(decoder,
int(decode_filters / 8),
'up3',
concat_with='pool1')
decoder = upproject(decoder,
int(decode_filters / 16),
'up4',
concat_with='conv1/relu')
if False:
decoder = upproject(decoder,
int(decode_filters / 32),
'up5',
concat_with='input_1')
# Extract depths (final layer)
conv3 = Conv2D(filters=1,
kernel_size=3,
strides=1,
padding='same',
name='conv3')(decoder)
output = Lambda(concat_output)([conv3, aux_output, has_locations])
output._keras_shape = output.get_shape().as_list()
# Create the model
model = Model(inputs=input, outputs=output)
else:
# Load model from file
if not existing.endswith('.h5'):
sys.exit(
'Please provide a correct model file when using [existing] argument.'
)
custom_objects = {
'BilinearUpSampling2D': BilinearUpSampling2D,
'depth_loss_function': depth_loss_function
}
model = load_model(existing, custom_objects=custom_objects)
print('\nExisting model loaded.\n')
print('Model created.')
return model
def conductance_model(input_shape=(11, 9, 1), filter_shape=[21, 1], num_filter=2, sum_over_space=True, fit_reversal=False):
reg_val = 1
v_leak = 0
v_exc = 60
v_inh = -30
g_leak = 1
filter_shape[1] = 1
pad_x = int((filter_shape[1] - 1))+2
pad_t = int((filter_shape[0] - 1))
# Define the input as a tensor with shape input_shape
image_in = Input(input_shape)
s1 = Lambda(lambda lam: lam[:, :, 0:-2, :])(image_in)
s2 = Lambda(lambda lam: lam[:, :, 1:-1, :])(image_in)
s3 = Lambda(lambda lam: lam[:, :, 2:, :])(image_in)
g1 = Conv2D(num_filter, filter_shape, strides=(1, 1), name='g1',
kernel_initializer=glorot_uniform(seed=None),
activation='relu',
kernel_regularizer=l1_reg_sqrt)(s1)
g2 = Conv2D(num_filter, filter_shape, strides=(1, 1), name='g2',
kernel_initializer=glorot_uniform(seed=None),
activation='relu',
kernel_regularizer=l1_reg_sqrt)(s2)
g3 = Conv2D(num_filter, filter_shape, strides=(1, 1), name='g3',
kernel_initializer=glorot_uniform(seed=None),
activation='relu',
kernel_regularizer=l1_reg_sqrt)(s3)
if fit_reversal:
expand_last = Lambda(lambda lam: K.expand_dims(lam, axis=-1))
squeeze_last = Lambda(lambda lam: K.squeeze(lam, axis=-1))
g1 = expand_last(g1)
g2 = expand_last(g2)
g3 = expand_last(g3)
numerator_in = Lambda(lambda inputs: K.concatenate(inputs, axis=4))([g1, g2, g3])
numerator = Conv3D(1, (1, 1, 1), strides=(1, 1, 1), name='create_numerator',
kernel_initializer=glorot_uniform(seed=None),
kernel_regularizer=l1_reg_sqrt,
use_bias=False)(numerator_in)
denominator = Lambda(lambda inputs: g_leak + inputs[0] + inputs[1] + inputs[2])([g1, g2, g3])
vm = Lambda(lambda inputs: inputs[0] / inputs[1])([numerator, denominator])
vm = squeeze_last(vm)
else:
g1_v_inh = Lambda(lambda lam: lam * v_inh)(g1)
g2_v_exc = Lambda(lambda lam: lam * v_exc)(g2)
g3_v_inh = Lambda(lambda lam: lam * v_inh)(g3)
numerator = Lambda(lambda inputs: inputs[0] + inputs[1] + inputs[2])([g1_v_inh, g2_v_exc, g3_v_inh])
denominator = Lambda(lambda inputs: g_leak + inputs[0] + inputs[1] + inputs[2])([g1, g2, g3])
vm = Lambda(lambda inputs: inputs[0] / inputs[1])([numerator, denominator])
vm_bias = BiasLayer()(vm)
vm_rect = Lambda(lambda lam: K.relu(lam))(vm_bias)
if sum_over_space:
conv_x_size = vm.get_shape().as_list()[2]
else:
conv_x_size = 1
combine_filters = Conv2D(1, (1, conv_x_size), strides=(1, 1), name='conv2',
kernel_initializer=glorot_uniform(seed=None),
kernel_regularizer=l1_reg_sqrt,
use_bias=False)(vm_rect)
# Create model
model = Model(inputs=image_in, outputs=combine_filters, name='conductance_model')
return model, pad_x, pad_t
def build(self, x_shape, c_shape=None):
input_shape = x_shape
x_inputs = Input(input_shape, name="%s_x_input" % self.name)
y_inputs = Input(input_shape, name="%s_y_input" % self.name)
concat_inputs = Concatenate(name="%s_concat_input" % self.name,
axis=-1)([x_inputs, y_inputs])
inputs = [x_inputs, y_inputs]
if c_shape is not None:
c_inputs = Input(c_shape, name="%s_c_input" % self.name)
inputs = [x_inputs, c_inputs, y_inputs]
wdout = concat_inputs
wodout = concat_inputs
nfilters = self.nfilters * 2
for il, stride in enumerate(self.strides):
conv = Convolution2D(nfilters,
kernel_size=self.kernel_size,
use_bias=self.use_bias,
name="%s_conv%d" % (self.name, il),
strides=(stride, stride),
padding='valid')
if il > 0:
bn = BatchNormalization(axis=1,
name="%s_bn%d" % (self.name, il))
wdout = bn(wdout)
wodout = bn(wodout)
do = Dropout(self.dropout_rate, name="%s_do%d" % (self.name, il))
act = LeakyReLU(0.2, name="%s_act%d" % (self.name, il))
wdout = act(do(conv(wdout)))
wodout = act(conv(wodout))
nfilters *= 2
if c_shape is not None:
nembed = nfilters // 2
c_bn0 = BatchNormalization(axis=-1,
name="%s_c_bn0" % self.name)(c_inputs)
c_dense = Dense(nembed,
use_bias=self.use_bias,
name="%s_c_dense" % self.name)(c_bn0)
c_bn = BatchNormalization(axis=-1,
name="%s_c_bn" % self.name)(c_dense)
c_rshp = Reshape((1, 1, nembed),
name="%s_c_rshp" % self.name)(c_bn)
ups = wdout.get_shape()
ups = (int(ups[1]), int(ups[2]))
c_up = UpSampling2D(size=ups, name="%s_c_up" % self.name)(c_rshp)
concat = Concatenate(name="%s_concat" % self.name, axis=-1)
wdout = concat([wdout, c_up])
wodout = concat([wodout, c_up])
if self.obo_layers is not None:
nch = self.obo_nch
if nch == 0:
nch = nfilters // 2
for il in range(self.obo_layers):
fc = Convolution2D(nch,
kernel_size=(1, 1),
use_bias=self.use_bias,
strides=(1, 1),
name="%s_fc%d" % (self.name, il))
wdout = fc(wdout)
wodout = fc(wodout)
bn = BatchNormalization(axis=1,
name="%s_obo_bn%d" % (self.name, il))
wdout = bn(wdout)
wodout = bn(wodout)
act = LeakyReLU(0.2, name="%s_obo_act%d" % (self.name, il))
wdout = act(wdout)
wodout = act(wodout)
if self.obo_nch_red != 0:
nch = max(self.obo_nch_min, nch // self.obo_nch_red)
flat = Flatten(name="%s_flat" % self.name)
wdout = flat(wdout)
wodout = flat(wodout)
dense = Dense(1, name="%s_dense" % self.name)
wdout = dense(wdout)
wodout = dense(wodout)
out = Activation("sigmoid", name="%s_out" % self.name)
wdout = out(wdout)
wodout = out(wodout)
wdooutputs = [wdout]
wodooutputs = [wodout]
if self.do_total:
sum_x = Lambda(
lambda x: K.sum(x, axis=(1, 2)), #output_shape=(1,),
name="%s_sum_x" % self.name)(x_inputs)
sum_y = Lambda(
lambda x: K.sum(x, axis=(1, 2)), #output_shape=(1,),
name="%s_sum_y" % self.name)(y_inputs)
concat_sum = [sum_x, sum_y]
print(sum_x.get_shape(), sum_y.get_shape())
if c_shape is not None:
c_flat = Flatten(name="%s_c_flt" % self.name)(c_bn)
concat_sum.append(c_flat)
concat_sum = Concatenate(name="%s_concat_sum" % self.name,
axis=-1)(concat_sum)
wdout = concat_sum
wodout = concat_sum
for il, nc in enumerate(self.total_layers):
t_dense = Dense(nembed,
use_bias=self.use_bias,
name="%s_t_dense%d" % (self.name, il))
t_bn = BatchNormalization(axis=-1,
name="%s_t_bn%d" % (self.name, il))
t_do = Dropout(self.dropout_rate,
name="%s_t_do%d" % (self.name, il))
t_act = LeakyReLU(0.2, name="%s_t_act%d" % (self.name, il))
wdout = t_act(t_do(t_bn(t_dense(wdout))))
wodout = t_act(t_bn(t_dense(wodout)))
dense = Dense(1, name="%s_t_dense" % self.name)
wdout = dense(wdout)
wodout = dense(wodout)
out = Activation("sigmoid", name="%s_t_out" % self.name)
wdout = out(wdout)
wodout = out(wodout)
wdooutputs.append(wdout)
wodooutputs.append(wodout)
model = (Model(inputs=inputs, outputs=wdooutputs),
Model(inputs=inputs, outputs=wodooutputs))
return model
def set_model(self):
'''
Initialisers
'''
weight_seed = None
kernel_initializer = initializers.glorot_uniform(seed = weight_seed)
bias_initializer = initializers.glorot_uniform(seed = weight_seed)
'''
Encoder
'''
# define input with 'channels_first'
input_encoder = Input(shape=self.input_shape, name='encoder_input')
x = Conv2D(self.filters,
self.kernel_size,
padding='same',
activation=None,
kernel_initializer=kernel_initializer,
bias_initializer=bias_initializer,
name='encoder_conv2D_1')(input_encoder)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2D(2*self.filters,
self.kernel_size,
padding='same',
activation=None,
kernel_initializer=kernel_initializer,
bias_initializer=bias_initializer,
name='encoder_conv2D_2')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
# take z_mean / z_log_var input shape
latent_input_shape = tuple(x.get_shape().as_list())
latent_width, latent_height = latent_input_shape[2:]
# separate dense layers for mu and log(sigma), both of size latent_dim
z_mean = Conv2D(self.latent_channels, kernel_size=(latent_width, latent_height), activation=None, kernel_initializer=kernel_initializer, bias_initializer=bias_initializer, name='encoder_z_mean')(x)
z_log_var = Conv2D(self.latent_channels, kernel_size=(latent_width, latent_height), activation=None, kernel_initializer=kernel_initializer, bias_initializer=bias_initializer, name='encoder_z_log_var')(x)
# sample from normal with z_mean and z_log_var
z = Lambda(self.sampling, name='encoder_z')([z_mean, z_log_var])
'''
Decoder
'''
# take encoder output shape
encoder_out_shape = tuple(z.get_shape().as_list())
# define rest of model
input_decoder = Input(shape=encoder_out_shape[1:], name='decoder_input')
x = Conv2DTranspose(2*self.filters,
kernel_size=(latent_width, latent_height),
padding='same',
activation=None,
kernel_initializer=kernel_initializer,
bias_initializer=bias_initializer,
name='decoder_conv2DT_1')(input_decoder)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2DTranspose(self.filters,
self.kernel_size,
activation=None,
padding='same',
kernel_initializer=kernel_initializer,
bias_initializer=bias_initializer,
name='decoder_conv2DT_2')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2DTranspose(self.filters,
self.kernel_size,
activation=None,
padding='same',
kernel_initializer=kernel_initializer,
bias_initializer=bias_initializer,
name='decoder_conv2DT_3')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2DTranspose(1,
self.kernel_size,
activation=None,
padding='same',
kernel_initializer=kernel_initializer,
bias_initializer=bias_initializer,
name='decoder_conv2DT_4')(x)
x = BatchNormalization()(x)
decoded_img = Activation('sigmoid')(x)
'''
Necessary definitions
'''
# For parent fitting function
self.encoder = Model(input_encoder, z)
self.decoder = Model(input_decoder, decoded_img)
self.model = Model(input_encoder, self.decoder(self.encoder(input_encoder)))
# For parent loss function
self.z_mean = z_mean
self.z_log_var = z_log_var
self.z = z
