def decoder_smate(encoder, timesteps, data_dim, pool_step):
input_ = encoder.output # input: (batch_size, timesteps, latent_dim)
out = ll.UpSampling1D(size=pool_step)(input_)
# 1D-CNN for reconstructing the spatial information
#cells = [rnn_cell(module_name) for _ in range(num_layers)]
#out = ll.RNN(cells, return_sequences=True)(out)
# temporal axis
if (module_name == 'gru'):
if (tf.test.is_gpu_available()):
out_t = ll.CuDNNGRU(hidden_dim, return_sequences=True)(out)
for i in range(num_layers - 1):
out_t = ll.CuDNNGRU(hidden_dim, return_sequences=True)(
out_t) # output: (batch_size, timesteps, hidden_dim)
else:
out_t = ll.GRU(hidden_dim, return_sequences=True)(out)
for i in range(num_layers - 1):
out_t = ll.GRU(hidden_dim, return_sequences=True)(
out_t) # output: (batch_size, timesteps, hidden_dim)
elif (module_name == 'lstm'):
if (tf.test.is_gpu_available()):
out_t = ll.CuDNNLSTM(hidden_dim, return_sequences=True)(out)
for i in range(num_layers - 1):
out_t = ll.CuDNNLSTM(hidden_dim, return_sequences=True)(
out_t) # output: (batch_size, timesteps, hidden_dim)
else:
out_t = ll.LSTM(hidden_dim, return_sequences=True)(out)
for i in range(num_layers - 1):
out_t = ll.LSTM(hidden_dim, return_sequences=True)(
out_t) # output: (batch_size, timesteps, hidden_dim)
out = ll.Dense(data_dim, activation='sigmoid')(out_t)
model = Model(encoder.input, out)
return model
def keras_model_fn(model_config, vocab_size, embedding_size, embeddings):
"""GPU version of Stacked Bi-LSTM and Bi-GRU with Two Fasttext
"""
## hyperparams
model_name = model_config['model_name']
num_class = model_config['num_class']
lstm_hs = model_config['lstm_hs']
gru_hs = model_config['gru_hs']
learning_rate = model_config['learning_rate']
## build model - , weights=[embeddings[1]]
inputs = ks.Input(shape=(None, ), dtype='int32', name='inputs')
embedded_sequences_ft1 = layers.Embedding(vocab_size,
embedding_size,
trainable=True,
mask_zero=False)(inputs)
embedded_sequences_ft2 = layers.Embedding(vocab_size,
embedding_size,
trainable=True,
mask_zero=False)(inputs)
concat_embed = layers.concatenate(
[embedded_sequences_ft1, embedded_sequences_ft2])
concat_embed = layers.SpatialDropout1D(0.5)(concat_embed)
x = layers.Bidirectional(layers.CuDNNLSTM(
lstm_hs, return_sequences=True))(concat_embed)
x, x_h, x_c = layers.Bidirectional(
layers.CuDNNGRU(gru_hs, return_sequences=True, return_state=True))(x)
x_1 = layers.GlobalMaxPool1D()(x)
x_2 = layers.GlobalAvgPool1D()(x)
x_out = layers.concatenate([x_1, x_2, x_h])
x_out = layers.BatchNormalization()(x_out)
outputs = layers.Dense(num_class, activation='softmax',
name='outputs')(x_out) # outputs
model = ks.Model(inputs, outputs, name=model_name)
## compile
model.compile(loss='categorical_crossentropy',
optimizer=ks.optimizers.Adam(lr=learning_rate,
clipnorm=.25,
beta_1=0.7,
beta_2=0.99),
metrics=[
'categorical_accuracy',
ks.metrics.TopKCategoricalAccuracy(k=3)
]) # metric what?
return model
def call(self, inputs):
z_mean, z_log_var = inputs
batch = tf.shape(z_mean)[0]
dim = tf.shape(z_mean)[1]
epsilon = tf.keras.backend.random_normal(shape=(batch, dim))
return z_mean + tf.exp(0.5 * z_log_var) * epsilon
original_dim = 128
intermediate_dim = 1024
latent_dim = 16
# Define encoder model.
original_inputs = tf.keras.Input(shape=(original_dim, 1), name='encoder_input')
input_err = Input(shape=(original_dim, 1))
x = layers.CuDNNGRU(intermediate_dim, return_sequences=False)(original_inputs)
z_mean = layers.Dense(latent_dim, name='z_mean')(x)
z_log_var = layers.Dense(latent_dim, name='z_log_var')(x)
z = Sampling()((z_mean, z_log_var))
encoder = tf.keras.Model(inputs=original_inputs, outputs=z, name='encoder')
# Define decoder model.
latent_inputs = tf.keras.Input(shape=(latent_dim, ), name='z_sampling')
x = layers.RepeatVector(original_dim)(latent_inputs)
x = layers.CuDNNGRU(intermediate_dim, return_sequences=True)(x)
outputs = layers.TimeDistributed(layers.Dense(1))(x)
decoder = tf.keras.Model(inputs=latent_inputs, outputs=outputs, name='decoder')
# Define VAE model.
outputs = decoder(z)
vae = tf.keras.Model(inputs=[original_inputs, input_err],
#Import
import os
os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2"
import tensorflow as tf
import tensorflow.keras.layers as KL
## Dataset
mnist = tf.keras.datasets.mnist
(x_train, y_train), (x_test, y_test) = mnist.load_data()
x_train, x_test = x_train / 255.0, x_test / 255.0
## Model
inputs = KL.Input(shape=(28, 28))
x = KL.CuDNNGRU(64)(inputs)
outputs = KL.Dense(10, activation="softmax")(x)
model = tf.keras.models.Model(inputs, outputs)
model.summary()
model.compile(optimizer="adam",
loss="sparse_categorical_crossentropy",
metrics=["acc"])
model.fit(x_train, y_train, epochs=5)
test_loss, test_acc = model.evaluate(x_test, y_test)
print("Loss: {0} - Acc: {1}".format(test_loss, test_acc))
def encoder_smate(in_shape, pool_step, d_prime, kernels=[8, 5, 3]):
input_ = Input(shape=in_shape) # input: (samples, L, input_dims)
L = in_shape[0]
# temporal axis
if (module_name == 'gru'):
if (tf.test.is_gpu_available()):
out_t = ll.CuDNNGRU(hidden_dim, return_sequences=True)(input_)
for i in range(num_layers - 1):
out_t = ll.CuDNNGRU(hidden_dim, return_sequences=True)(
out_t) # output: (batch_size, timesteps, hidden_dim)
else:
out_t = ll.GRU(hidden_dim, return_sequences=True)(input_)
for i in range(num_layers - 1):
out_t = ll.GRU(hidden_dim, return_sequences=True)(
out_t) # output: (batch_size, timesteps, hidden_dim)
elif (module_name == 'lstm'):
if (tf.test.is_gpu_available()):
out_t = ll.CuDNNLSTM(hidden_dim, return_sequences=True)(input_)
for i in range(num_layers - 1):
out_t = ll.CuDNNLSTM(hidden_dim, return_sequences=True)(
out_t) # output: (batch_size, timesteps, hidden_dim)
else:
out_t = ll.LSTM(hidden_dim, return_sequences=True)(input_)
for i in range(num_layers - 1):
out_t = ll.LSTM(hidden_dim, return_sequences=True)(
out_t) # output: (batch_size, timesteps, hidden_dim)
# 1D-CNN
out_s = spatial_dynamic_block(input_, kernels[0], d_prime)
out_s = Conv1D(128,
kernels[0],
padding='same',
kernel_initializer='he_uniform')(input_)
out_s = BatchNormalization()(out_s)
out_s = Activation('relu')(out_s)
out_s = spatial_dynamic_block(out_s, kernels[1], 8)
out_s = Conv1D(256,
kernels[1],
padding='same',
kernel_initializer='he_uniform')(out_s)
out_s = BatchNormalization()(out_s)
out_s = Activation('relu')(out_s)
out_s = spatial_dynamic_block(out_s, kernels[2], 16)
out_s = Conv1D(128,
kernels[2],
padding='same',
kernel_initializer='he_uniform')(out_s)
out_s = BatchNormalization()(out_s)
out_s = Activation('relu')(out_s) # L * D
#reduce latent space dimension (t & s axis)
out_t = AveragePooling1D(pool_size=pool_step, strides=None,
padding='same')(out_t)
out_s = AveragePooling1D(pool_size=pool_step, strides=None,
padding='same')(out_s) # L' * D
out = ll.Concatenate(axis=-1)([out_t, out_s]) # (samples, L', 128*4 + 128)
out = Dense(128)(out)
#out = Dense(128)(out_s)
out = BatchNormalization()(out)
out = ll.LeakyReLU()(out)
out = Dense(128)(out)
out = BatchNormalization()(out) # (samples, L', 128)
model = Model(inputs=input_, outputs=out)
return model
def encoder_smate_rdp(in_shape, pool_step):
input_ = Input(shape=in_shape) # input: (samples, L, input_dims)
L = in_shape[0]
D = in_shape[1]
# temporal axis
if (module_name == 'gru'):
if (tf.test.is_gpu_available()):
out_t = ll.CuDNNGRU(hidden_dim, return_sequences=True)(input_)
for i in range(num_layers - 1):
out_t = ll.CuDNNGRU(hidden_dim, return_sequences=True)(
out_t) # output: (batch_size, timesteps, hidden_dim)
else:
out_t = ll.GRU(hidden_dim, return_sequences=True)(input_)
for i in range(num_layers - 1):
out_t = ll.GRU(hidden_dim, return_sequences=True)(
out_t) # output: (batch_size, timesteps, hidden_dim)
elif (module_name == 'lstm'):
if (tf.test.is_gpu_available()):
out_t = ll.CuDNNLSTM(hidden_dim, return_sequences=True)(input_)
for i in range(num_layers - 1):
out_t = ll.CuDNNLSTM(hidden_dim, return_sequences=True)(
out_t) # output: (batch_size, timesteps, hidden_dim)
else:
out_t = ll.LSTM(hidden_dim, return_sequences=True)(input_)
for i in range(num_layers - 1):
out_t = ll.LSTM(hidden_dim, return_sequences=True)(
out_t) # output: (batch_size, timesteps, hidden_dim)
# 1D-CNN
group_size = int(D * 1.5 / 3)
in_s = []
for i in range(3):
idx_list = rd.sample(range(0, D), group_size)
idx_array = np.array(idx_list)
in_s_i = input_[:, :, i:i + group_size]
in_s.append(in_s_i)
out_s_11 = Conv1D(128, 8, padding='same',
kernel_initializer='he_uniform')(in_s[0])
out_s_11 = BatchNormalization()(out_s_11)
out_s_11 = Activation('relu')(out_s_11)
out_s_12 = Conv1D(128, 8, padding='same',
kernel_initializer='he_uniform')(in_s[1])
out_s_12 = BatchNormalization()(out_s_12)
out_s_12 = Activation('relu')(out_s_12)
out_s_13 = Conv1D(128, 8, padding='same',
kernel_initializer='he_uniform')(in_s[2])
out_s_13 = BatchNormalization()(out_s_13)
out_s_13 = Activation('relu')(out_s_13)
out_s_1 = [out_s_11, out_s_12, out_s_13]
#out_s = K.concatenate((out_s_11, out_s_12, out_s_13), axis=0)
conv1D_2 = Conv1D(256, 5, padding='same', kernel_initializer='he_uniform')
conv1D_3 = Conv1D(128, 3, padding='same', kernel_initializer='he_uniform')
batch_norm1 = BatchNormalization()
batch_norm2 = BatchNormalization()
activ_relu1 = Activation('relu')
activ_relu2 = Activation('relu')
s_outs = []
for s in out_s_1:
s_out = conv1D_2(s)
s_out = batch_norm1(s_out)
s_out = activ_relu1(s_out)
s_out = conv1D_3(s_out)
s_out = batch_norm2(s_out)
s_out = activ_relu2(s_out)
s_out = AveragePooling1D(pool_size=pool_step,
strides=None,
padding='same')(s_out) # L * D
s_outs.append(s_out)
out_s = ll.Concatenate(axis=-1)(s_outs) # L * 3D
#reduce latent space dimension (t & s axis)
out_t = AveragePooling1D(pool_size=pool_step, strides=None,
padding='same')(out_t)
out = ll.Concatenate(axis=-1)([out_t, out_s]) # 1 * 4D
out = Dense(128)(out)
out = BatchNormalization()(out)
out = ll.LeakyReLU()(out)
out = Dense(128)(out)
out = BatchNormalization()(out) # (samples, L', 128)
model = Model(inputs=input_, outputs=out)
return model