def decoder_smate(encoder, timesteps, data_dim, pool_step): input_ = encoder.output # input: (batch_size, timesteps, latent_dim) out = ll.UpSampling1D(size=pool_step)(input_) # 1D-CNN for reconstructing the spatial information #cells = [rnn_cell(module_name) for _ in range(num_layers)] #out = ll.RNN(cells, return_sequences=True)(out) # temporal axis if (module_name == 'gru'): if (tf.test.is_gpu_available()): out_t = ll.CuDNNGRU(hidden_dim, return_sequences=True)(out) for i in range(num_layers - 1): out_t = ll.CuDNNGRU(hidden_dim, return_sequences=True)( out_t) # output: (batch_size, timesteps, hidden_dim) else: out_t = ll.GRU(hidden_dim, return_sequences=True)(out) for i in range(num_layers - 1): out_t = ll.GRU(hidden_dim, return_sequences=True)( out_t) # output: (batch_size, timesteps, hidden_dim) elif (module_name == 'lstm'): if (tf.test.is_gpu_available()): out_t = ll.CuDNNLSTM(hidden_dim, return_sequences=True)(out) for i in range(num_layers - 1): out_t = ll.CuDNNLSTM(hidden_dim, return_sequences=True)( out_t) # output: (batch_size, timesteps, hidden_dim) else: out_t = ll.LSTM(hidden_dim, return_sequences=True)(out) for i in range(num_layers - 1): out_t = ll.LSTM(hidden_dim, return_sequences=True)( out_t) # output: (batch_size, timesteps, hidden_dim) out = ll.Dense(data_dim, activation='sigmoid')(out_t) model = Model(encoder.input, out) return model
def keras_model_fn(model_config, vocab_size, embedding_size, embeddings): """GPU version of Stacked Bi-LSTM and Bi-GRU with Two Fasttext """ ## hyperparams model_name = model_config['model_name'] num_class = model_config['num_class'] lstm_hs = model_config['lstm_hs'] gru_hs = model_config['gru_hs'] learning_rate = model_config['learning_rate'] ## build model - , weights=[embeddings[1]] inputs = ks.Input(shape=(None, ), dtype='int32', name='inputs') embedded_sequences_ft1 = layers.Embedding(vocab_size, embedding_size, trainable=True, mask_zero=False)(inputs) embedded_sequences_ft2 = layers.Embedding(vocab_size, embedding_size, trainable=True, mask_zero=False)(inputs) concat_embed = layers.concatenate( [embedded_sequences_ft1, embedded_sequences_ft2]) concat_embed = layers.SpatialDropout1D(0.5)(concat_embed) x = layers.Bidirectional(layers.CuDNNLSTM( lstm_hs, return_sequences=True))(concat_embed) x, x_h, x_c = layers.Bidirectional( layers.CuDNNGRU(gru_hs, return_sequences=True, return_state=True))(x) x_1 = layers.GlobalMaxPool1D()(x) x_2 = layers.GlobalAvgPool1D()(x) x_out = layers.concatenate([x_1, x_2, x_h]) x_out = layers.BatchNormalization()(x_out) outputs = layers.Dense(num_class, activation='softmax', name='outputs')(x_out) # outputs model = ks.Model(inputs, outputs, name=model_name) ## compile model.compile(loss='categorical_crossentropy', optimizer=ks.optimizers.Adam(lr=learning_rate, clipnorm=.25, beta_1=0.7, beta_2=0.99), metrics=[ 'categorical_accuracy', ks.metrics.TopKCategoricalAccuracy(k=3) ]) # metric what? return model
def call(self, inputs): z_mean, z_log_var = inputs batch = tf.shape(z_mean)[0] dim = tf.shape(z_mean)[1] epsilon = tf.keras.backend.random_normal(shape=(batch, dim)) return z_mean + tf.exp(0.5 * z_log_var) * epsilon original_dim = 128 intermediate_dim = 1024 latent_dim = 16 # Define encoder model. original_inputs = tf.keras.Input(shape=(original_dim, 1), name='encoder_input') input_err = Input(shape=(original_dim, 1)) x = layers.CuDNNGRU(intermediate_dim, return_sequences=False)(original_inputs) z_mean = layers.Dense(latent_dim, name='z_mean')(x) z_log_var = layers.Dense(latent_dim, name='z_log_var')(x) z = Sampling()((z_mean, z_log_var)) encoder = tf.keras.Model(inputs=original_inputs, outputs=z, name='encoder') # Define decoder model. latent_inputs = tf.keras.Input(shape=(latent_dim, ), name='z_sampling') x = layers.RepeatVector(original_dim)(latent_inputs) x = layers.CuDNNGRU(intermediate_dim, return_sequences=True)(x) outputs = layers.TimeDistributed(layers.Dense(1))(x) decoder = tf.keras.Model(inputs=latent_inputs, outputs=outputs, name='decoder') # Define VAE model. outputs = decoder(z) vae = tf.keras.Model(inputs=[original_inputs, input_err],
#Import import os os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2" import tensorflow as tf import tensorflow.keras.layers as KL ## Dataset mnist = tf.keras.datasets.mnist (x_train, y_train), (x_test, y_test) = mnist.load_data() x_train, x_test = x_train / 255.0, x_test / 255.0 ## Model inputs = KL.Input(shape=(28, 28)) x = KL.CuDNNGRU(64)(inputs) outputs = KL.Dense(10, activation="softmax")(x) model = tf.keras.models.Model(inputs, outputs) model.summary() model.compile(optimizer="adam", loss="sparse_categorical_crossentropy", metrics=["acc"]) model.fit(x_train, y_train, epochs=5) test_loss, test_acc = model.evaluate(x_test, y_test) print("Loss: {0} - Acc: {1}".format(test_loss, test_acc))
def encoder_smate(in_shape, pool_step, d_prime, kernels=[8, 5, 3]): input_ = Input(shape=in_shape) # input: (samples, L, input_dims) L = in_shape[0] # temporal axis if (module_name == 'gru'): if (tf.test.is_gpu_available()): out_t = ll.CuDNNGRU(hidden_dim, return_sequences=True)(input_) for i in range(num_layers - 1): out_t = ll.CuDNNGRU(hidden_dim, return_sequences=True)( out_t) # output: (batch_size, timesteps, hidden_dim) else: out_t = ll.GRU(hidden_dim, return_sequences=True)(input_) for i in range(num_layers - 1): out_t = ll.GRU(hidden_dim, return_sequences=True)( out_t) # output: (batch_size, timesteps, hidden_dim) elif (module_name == 'lstm'): if (tf.test.is_gpu_available()): out_t = ll.CuDNNLSTM(hidden_dim, return_sequences=True)(input_) for i in range(num_layers - 1): out_t = ll.CuDNNLSTM(hidden_dim, return_sequences=True)( out_t) # output: (batch_size, timesteps, hidden_dim) else: out_t = ll.LSTM(hidden_dim, return_sequences=True)(input_) for i in range(num_layers - 1): out_t = ll.LSTM(hidden_dim, return_sequences=True)( out_t) # output: (batch_size, timesteps, hidden_dim) # 1D-CNN out_s = spatial_dynamic_block(input_, kernels[0], d_prime) out_s = Conv1D(128, kernels[0], padding='same', kernel_initializer='he_uniform')(input_) out_s = BatchNormalization()(out_s) out_s = Activation('relu')(out_s) out_s = spatial_dynamic_block(out_s, kernels[1], 8) out_s = Conv1D(256, kernels[1], padding='same', kernel_initializer='he_uniform')(out_s) out_s = BatchNormalization()(out_s) out_s = Activation('relu')(out_s) out_s = spatial_dynamic_block(out_s, kernels[2], 16) out_s = Conv1D(128, kernels[2], padding='same', kernel_initializer='he_uniform')(out_s) out_s = BatchNormalization()(out_s) out_s = Activation('relu')(out_s) # L * D #reduce latent space dimension (t & s axis) out_t = AveragePooling1D(pool_size=pool_step, strides=None, padding='same')(out_t) out_s = AveragePooling1D(pool_size=pool_step, strides=None, padding='same')(out_s) # L' * D out = ll.Concatenate(axis=-1)([out_t, out_s]) # (samples, L', 128*4 + 128) out = Dense(128)(out) #out = Dense(128)(out_s) out = BatchNormalization()(out) out = ll.LeakyReLU()(out) out = Dense(128)(out) out = BatchNormalization()(out) # (samples, L', 128) model = Model(inputs=input_, outputs=out) return model
def encoder_smate_rdp(in_shape, pool_step): input_ = Input(shape=in_shape) # input: (samples, L, input_dims) L = in_shape[0] D = in_shape[1] # temporal axis if (module_name == 'gru'): if (tf.test.is_gpu_available()): out_t = ll.CuDNNGRU(hidden_dim, return_sequences=True)(input_) for i in range(num_layers - 1): out_t = ll.CuDNNGRU(hidden_dim, return_sequences=True)( out_t) # output: (batch_size, timesteps, hidden_dim) else: out_t = ll.GRU(hidden_dim, return_sequences=True)(input_) for i in range(num_layers - 1): out_t = ll.GRU(hidden_dim, return_sequences=True)( out_t) # output: (batch_size, timesteps, hidden_dim) elif (module_name == 'lstm'): if (tf.test.is_gpu_available()): out_t = ll.CuDNNLSTM(hidden_dim, return_sequences=True)(input_) for i in range(num_layers - 1): out_t = ll.CuDNNLSTM(hidden_dim, return_sequences=True)( out_t) # output: (batch_size, timesteps, hidden_dim) else: out_t = ll.LSTM(hidden_dim, return_sequences=True)(input_) for i in range(num_layers - 1): out_t = ll.LSTM(hidden_dim, return_sequences=True)( out_t) # output: (batch_size, timesteps, hidden_dim) # 1D-CNN group_size = int(D * 1.5 / 3) in_s = [] for i in range(3): idx_list = rd.sample(range(0, D), group_size) idx_array = np.array(idx_list) in_s_i = input_[:, :, i:i + group_size] in_s.append(in_s_i) out_s_11 = Conv1D(128, 8, padding='same', kernel_initializer='he_uniform')(in_s[0]) out_s_11 = BatchNormalization()(out_s_11) out_s_11 = Activation('relu')(out_s_11) out_s_12 = Conv1D(128, 8, padding='same', kernel_initializer='he_uniform')(in_s[1]) out_s_12 = BatchNormalization()(out_s_12) out_s_12 = Activation('relu')(out_s_12) out_s_13 = Conv1D(128, 8, padding='same', kernel_initializer='he_uniform')(in_s[2]) out_s_13 = BatchNormalization()(out_s_13) out_s_13 = Activation('relu')(out_s_13) out_s_1 = [out_s_11, out_s_12, out_s_13] #out_s = K.concatenate((out_s_11, out_s_12, out_s_13), axis=0) conv1D_2 = Conv1D(256, 5, padding='same', kernel_initializer='he_uniform') conv1D_3 = Conv1D(128, 3, padding='same', kernel_initializer='he_uniform') batch_norm1 = BatchNormalization() batch_norm2 = BatchNormalization() activ_relu1 = Activation('relu') activ_relu2 = Activation('relu') s_outs = [] for s in out_s_1: s_out = conv1D_2(s) s_out = batch_norm1(s_out) s_out = activ_relu1(s_out) s_out = conv1D_3(s_out) s_out = batch_norm2(s_out) s_out = activ_relu2(s_out) s_out = AveragePooling1D(pool_size=pool_step, strides=None, padding='same')(s_out) # L * D s_outs.append(s_out) out_s = ll.Concatenate(axis=-1)(s_outs) # L * 3D #reduce latent space dimension (t & s axis) out_t = AveragePooling1D(pool_size=pool_step, strides=None, padding='same')(out_t) out = ll.Concatenate(axis=-1)([out_t, out_s]) # 1 * 4D out = Dense(128)(out) out = BatchNormalization()(out) out = ll.LeakyReLU()(out) out = Dense(128)(out) out = BatchNormalization()(out) # (samples, L', 128) model = Model(inputs=input_, outputs=out) return model