def __init__(self, vocab_len=50000, embed_dims=128, hidden_dims=256, rnn_type="LSTM", batch_size=128): super(Encoder, self).__init__() self.batch_size = batch_size self.hidden_dims = hidden_dims self.embedding = layers.Embedding(input_dim=vocab_len, output_dim=embed_dims, input_length=400) if rnn_type == "LSTM": self.recurrent_layer = layers.Bidirectional( layers.LSTM(units=hidden_dims, activation='tanh', return_sequences=True, return_state=True, recurrent_initializer='glorot_uniform')) if rnn_type == "GRU": self.recurrent_layer = layers.Bidirectional( layers.GRU(units=hidden_dims, activation='tanh', return_sequences=True, return_state=True, recurrent_initializer='glorot_uniform')) if rnn_type == "RUM": raise NotImplementedError # Layer used to reduce output from bidirectional RNN layer. # Referenced https://github.com/abisee/pointer-generator to # examine their approach to this process, and adapted the # approach here. self.reduce_layer = layers.Dense(units=hidden_dims, activation='relu')
def __init__(self, config): super(PredictorDetRNN, self).__init__() self.rnn_type = config.rnn_type # Layers self.embedding = layers.Dense(config.emb_size, activation=config.activation_func) if self.rnn_type == "lstm": logging.info("Using LSTM RNNs") self.rnn = layers.LSTM(config.enc_hidden_size, return_sequences=False, return_state=True, activation='tanh', dropout=config.dropout_rate) else: logging.info("Using GRU RNNs") self.rnn = layers.GRU(config.enc_hidden_size, return_sequences=False, return_state=True, activation='tanh', dropout=config.dropout_rate) self.dropout = layers.Dropout(config.dropout_rate, name="dropout_decode") self.decoder = layers.Dense(config.P, activation=tf.identity) # Loss function self.loss_fun = losses.LogCosh()
def __init__(self, model_type: str, vocab_size: int, embed_dim: int, hidden_size: int = 128, training: bool = False): super(MyAdvancedModel, self).__init__() self.hidden_size = hidden_size self.num_classes = len(ID_TO_CLASS) self.model_type = model_type self.embeddings = tf.Variable(tf.random.normal((vocab_size, embed_dim))) if model_type == 'over': self.decoder = layers.Dense(units=self.num_classes) self.omegas1 = tf.Variable(tf.random.normal((hidden_size*2, hidden_size))) self.omegas2 = tf.Variable(tf.random.normal((hidden_size, (int)(hidden_size/2)))) self.omegas3 = tf.Variable(tf.random.normal(((int)(hidden_size/2), 1))) self.biDirection = layers.Bidirectional(tf.keras.layers.GRU(units=hidden_size, return_sequences=True, activation='tanh', recurrent_activation="tanh")) self.batch_normed = layers.BatchNormalization(trainable=True) elif model_type == 'sattn': self.biDirection = layers.Bidirectional(tf.keras.layers.GRU(units=hidden_size, return_sequences=True, activation='tanh', recurrent_activation="tanh")) self.decoder = layers.GRU(units=self.num_classes, return_sequences=True, activation='tanh') else: print ("CONV") self.decoder = layers.Dense(units=self.num_classes) self.biDirection = layers.Bidirectional(tf.keras.layers.GRU(units=hidden_size, return_sequences=True, activation='tanh', recurrent_activation="tanh")) self.conv = layers.Conv1D(filters = self.hidden_size, kernel_size = 3) self.omegas = tf.Variable(tf.random.normal((hidden_size, (int)(hidden_size/2))))
def __init__(self, input_dim: int, num_layers: int): super(GruSequenceToVector, self).__init__(input_dim) # TODO(students): start self.num_layers = num_layers self.hidden_layer = layers.GRU( self._input_dim, activation='tanh', return_sequences=True ) # Using tanh activation for each layer , and using return_sequences to capture current representation at each step.
def regression_model(input_dim = 31,tcn = False): if not tcn: from tensorflow.keras import layers, models import tensorflow as tf model = models.Sequential() model.add(layers.GRU(128, return_sequences=True, input_shape=(None, input_dim))) model.add(layers.TimeDistributed(layers.Dense(13, activation='linear'))) model.summary() model.compile(loss='mse', optimizer=tf.keras.optimizers.Adam(lr=0.01)) else: import tcn import keras from keras.layers import Dense,TimeDistributed from keras.models import Input, Model from tcn import TCN i = Input(batch_shape=(None, None, input_dim)) o = TCN(return_sequences=True)(i) # The TCN layers are here. o = TimeDistributed(Dense(13, activation='linear'))(o) model = Model(inputs=[i], outputs=[o]) model.summary() model.compile(loss='mse', optimizer= keras.optimizers.Adam(lr=0.01)) return model
def build_transactions_rnn(transactions_cat_features, embedding_projections, product_col_name='product', rnn_units=128, classifier_units=32, optimizer=None): if not optimizer: optimizer = keras.optimizers.Adam(lr=1e-3) inputs = [] cat_embeds = [] for feature_name in transactions_cat_features: inp = L.Input(shape=(None, ), dtype='uint32', name=f'input_{feature_name}') inputs.append(inp) source_size, projection = embedding_projections[feature_name] emb = L.Embedding(source_size + 1, projection, trainable=True, mask_zero=False, name=f'embedding_{feature_name}')(inp) cat_embeds.append(emb) # product feature inp = L.Input(shape=(1, ), dtype='uint32', name=f'input_product') inputs.append(inp) source_size, projection = embedding_projections['product'] product_emb = L.Embedding(source_size + 1, projection, trainable=True, mask_zero=False, name=f'embedding_product')(inp) product_emb_reshape = L.Reshape((projection, ))(product_emb) concated_cat_embeds = L.concatenate(cat_embeds) dropout_embeds = L.SpatialDropout1D(0.05)(concated_cat_embeds) sequences = L.Bidirectional(L.GRU(units=rnn_units, return_sequences=True))(dropout_embeds) pooled_avg_sequences = L.GlobalAveragePooling1D()(sequences) pooled_max_sequences = L.GlobalMaxPooling1D()(sequences) #add dropout=0.5 concated = L.concatenate( [pooled_avg_sequences, pooled_max_sequences, product_emb_reshape]) dense_intermediate = L.Dense(classifier_units, activation='relu', kernel_regularizer=keras.regularizers.L1L2( 1e-7, 1e-5))(concated) proba = L.Dense(1, activation='sigmoid')(dense_intermediate) model = Model(inputs=inputs, outputs=proba) model.compile(loss='binary_crossentropy', optimizer=optimizer) return model
def get_gru(dim, act=None, is_bidirectional=False, merge_mode="sum"): layer = layers.GRU(dim, activation=act, return_sequences=True, kernel_initializer=kernel_initializer, kernel_regularizer=kernel_regularizer) if is_bidirectional: layer = layers.Bidirectional(layer, merge_mode=merge_mode) return layer
def define_recurrent_encoder(timesteps, features, latent_dim): ''' Define recurrent encoder args: dimensions timesteps, feautres. latent dim: hyper-parameter return: recurrent encoder model ''' recurrent_encoder = tfkm.Sequential([ tfkl.Masking(mask_value=-1., input_shape=(timesteps, features)), tfkl.Dropout(0.2), tfkl.GRU(42, return_sequences = True), tfkl.GRU(latent_dim) ]) return recurrent_encoder
def __init__(self, K, conv_channels, pool_size, projections, highway_units, highway_nums, rnn_units, name='cbhg'): super(CBHG, self).__init__(name=name) self.K = K self.conv_channels = conv_channels self.pool_size = pool_size self.projections = projections self.highway_units = highway_units self.highway_nums = highway_nums self.rnn_units = rnn_units self.conv_banks = [ConvBlock('cab', '1D', conv_channels, k, name='conv_banks') for k in range(1, K + 1)] self.pool = layers.MaxPool1D(pool_size, strides=1, padding='same', name='pool') acts = ['relu'] * (len(projections) - 1) + [None] self.conv_projections = [ConvBlock('cab', '1D', c, 3, None, a, name='conv_projs') for c, a in zip(projections, acts)] self.highway_nets = [HighwayNet(highway_units, name='highway') for _ in range(highway_nums)] self.gru_layer = layers.Bidirectional(layers.GRU(rnn_units, return_sequences=True), name='bigru') self.supports_masking = True
def build_rnn_model(embedding_dim, vocabulary_size, recurrent_type='SimpleRNN', n_rnn_units=64, n_rnn_layers=1, bidirect=True): tf.random.set_seed(123) model = tf.keras.Sequential() model.add( tkl.Embedding(input_dim=vocabulary_size, output_dim=embedding_dim, name='embedding_layer')) for i in range(n_rnn_layers): return_sequences = (i < n_rnn_layers - 1) if recurrent_type == 'SimpleRNN': recurrent_layer = tkl.SimpleRNN( units=n_rnn_units, return_sequences=return_sequences, name='simple_rnn_layer{}'.format(i)) elif recurrent_type == 'LSTM': recurrent_layer = tkl.LSTM(units=n_rnn_units, return_sequences=return_sequences, name='lstm_layer{}'.format(i)) elif recurrent_type == 'GRU': recurrent_layer = tkl.GRU(units=n_rnn_units, return_seq=return_sequences, name='gru_layer{}'.format(i)) if bidirect: recurrent_layer = tkl.Bidirectional(recurrent_layer, name='bidirect_' + recurrent_layer.name) model.add(recurrent_layer) model.add(tkl.Dense(64, activation='relu')) model.add(tkl.Dense(1, activation='sigmoid')) return model
def __init__(self,vocab_size,emb_size,hid_size,dropout=0.1,recurrent_dropout=0.2): super(WordRNN,self).__init__() self.embed = L.Embedding(vocab_size,emb_size) self.gru = L.GRU(hid_size,dropout=dropout,recurrent_dropout=recurrent_dropout) self.bi_gru = L.Bidirectional(self.gru) self.fc = L.Dense(1)
def __init__(self, vocab_size, max_message_length, embed_dim, hidden_dim, vision_module, flexible_message_length=False, activation='linear', n_distractors=1, image_dim=32): super(Receiver, self).__init__(vocab_size, max_message_length, embed_dim, hidden_dim, vision_module, flexible_message_length, VtoH_activation=activation) self.n_distractors = n_distractors self.image_dim = image_dim # if message length fixed, 0 counts as a standard symbol and no masking is applied self.language_module = Sequential([ layers.Embedding(vocab_size, embed_dim, name='embedding'), layers.GRU(hidden_dim, name='GRU_layer') ]) # GRU linear by default self.__build()
def build_model_c_rnn(model_input): x_input = layers.Input(model_input) # -------Convolutional blocks------- # # First convolutional block x = layers.Conv2D(68, (3, 3), strides=(1, 1), padding='same', name='conv_1')(x_input) x = layers.BatchNormalization()(x) x = layers.Activation('relu')(x) x = layers.MaxPooling2D((2, 2), strides=(1, 1))(x) x = layers.Dropout(0.1)(x) # Second convolutional block x = layers.Conv2D(137, (3, 3), strides=(1, 1), padding='same', name='conv_2')(x) x = layers.BatchNormalization()(x) x = layers.Activation('relu')(x) x = layers.MaxPooling2D((3, 3), strides=(1, 1))(x) x = layers.Dropout(0.1)(x) # Third convolutional block x = layers.Conv2D(137, (3, 3), strides=(1, 1), padding='same', name='conv_3')(x) x = layers.BatchNormalization()(x) x = layers.Activation('relu')(x) x = layers.MaxPooling2D((4, 4), strides=(1, 1))(x) x = layers.Dropout(0.1)(x) # Fourth convolutional block x = layers.Conv2D(137, (3, 3), strides=(1, 1), padding='same', name='conv_4')(x) x = layers.BatchNormalization()(x) x = layers.Activation('relu')(x) x = layers.MaxPooling2D((4, 4), strides=(1, 1))(x) x = layers.Dropout(0.1)(x) # -------Recurrent Block------- # # GRU layer lstm = layers.GRU(68)(x[:, :, :, 0]) # Softmax Output output = layers.Dense(8, activation='softmax', name='preds')(lstm) model_output = output model = tf.keras.models.Model(x_input, model_output) opt = Adam(lr=0.001) # opt = tf.keras.optimizers.RMSprop(lr=0.0005) # Optimizer model.compile(loss='categorical_crossentropy', optimizer=opt, metrics=['accuracy']) print(model.summary()) return model
def Build_RNN_layer(Parametre_layer): if Parametre_layer["type_cell"] == "LSTM": cell = L.LSTM(units=Parametre_layer["units"], dropout=Parametre_layer["dropout"], recurrent_dropout=Parametre_layer["recurrent_dropout"], return_sequences=True, return_state=True, stateful=Parametre_layer["stateful"], unroll=Parametre_layer["unroll"]) elif Parametre_layer["type_cell"] == "CuDNNGRU": cell = CuDNNGRU(units=Parametre_layer["units"], dropout=Parametre_layer["dropout"], recurrent_dropout=Parametre_layer["recurrent_dropout"], return_sequences=True, return_state=True, stateful=Parametre_layer["stateful"], unroll=Parametre_layer["unroll"]) elif Parametre_layer["type_cell"] == "GRU": cell = L.GRU(units=Parametre_layer["units"], return_sequences=True, return_state=True, stateful=Parametre_layer["stateful"]) else: #by default CuDNNLSTM cell = CuDNNLSTM(units=Parametre_layer["units"], return_sequences=True, return_state=True, stateful=Parametre_layer["stateful"]) return cell
def __init__(self): super(LSTM, self).__init__() # Define a Masking Layer with -1 as mask. self.norm = x = layers.BatchNormalization() self.CNN = tf.keras.applications.EfficientNetB0( include_top=False, weights='imagenet', input_tensor=None, input_shape=None, pooling=None) self.CNN.trainable = train_CNN self.dense = layers.Dense(LSTM_input_units) # Define a LSTM layer to be applied over the Masking layer. # Dynamic computation will automatically be performed to ignore -1 values. self.lstm1 = layers.Bidirectional(layers.GRU(units=LSTM1_output_units, return_sequences=True,dropout=dropout)) self.lstm2 = layers.Bidirectional(layers.GRU(units=LSTM2_output_units, return_sequences=True,dropout=dropout)) self.lstm3 = layers.Bidirectional(layers.GRU(units=LSTM3_output_units, return_sequences=True,dropout=dropout)) # Output fully connected layer (5 classes). self.out = layers.Dense(num_classes+1)
def build_classifier_model(embedding_matrix): input = layers.Input(shape=(MAX_SEQ_LEN, ), dtype=np.int32) embedding_layer = layers.Embedding(VOCAB_SIZE + 1, EMBEDDING_DIMS, weights=[embedding_matrix], trainable=False) embedded_input = embedding_layer(input) gru_output = layers.Bidirectional( layers.GRU(HIDDEN_UNITS, return_sequences=True))(embedded_input) gru_output = layers.Bidirectional(layers.GRU(HIDDEN_UNITS))(gru_output) prob = layers.Dense(NUM_OUTPUTS, activation='sigmoid')(gru_output) bigru_model = Model(input, prob) print('generated bigru model...') return bigru_model
def main(): num_samples = 1000 num_features = 7 X = np.random.rand(num_samples, 2, 2, 3, 5) y = np.random.rand(num_samples, 2, 2) layer = GRU(512) batch_size = 30 timesteps = 20 features = 15 out = layer(np.random.rand([batch_size, timesteps, features])) model = Sequential([ keras.Input(shape=(None, num_features)), layers.GRU(512), layers.BatchNormalization() ]) model = Sequential() model.add(Flatten()) model.add(Dense(12, input_dim=num_features, activation='relu')) model.add(Dense(8, activation='relu')) model.add(Dense(1, activation='sigmoid')) model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy']) # fit the keras model on the dataset model.fit(X, y, epochs=15, batch_size=10) # evaluate the keras model _, accuracy = model.evaluate(X, y) print('Accuracy: %.2f' % (accuracy))
def __init__(self, task): self.task = task options = self.build_options() self.options = options self.outputs = {} self.my_losses = {} with tf.compat.v1.variable_scope('vqa_quality'): self.q_embed = layers.Embedding( options['vocab_size'], options['embed_size'], input_length=options['question_len']) self.q_gru = layers.GRU(options['q_encoded_size'], dropout=options['drop_prob']) self.topdown_tanh = layers.Conv2D(options['topdown_gated_size'], 1, activation='tanh') self.topdown_sig = layers.Conv2D(options['topdown_gated_size'], 1, activation='sigmoid') self.topdown_multiply = layers.Multiply() self.topdown_conv = layers.Conv2D(1, 1) self.q_gated_tanh = GatedTanh(options['q_encoded_size'], options['q_encoded_size']) self.img_gated_tanh = GatedTanh(options['img_feat_dim'], options['q_encoded_size']) self.VQ_joint = layers.Multiply() self.ans_gated_tanh = GatedTanh(options['q_encoded_size'], options['ans_fc_dim']) self.ans_dense = layers.Dense( 1, input_shape=(options['ans_fc_dim'], ), activation='sigmoid')
def main(): mirrored_strategy = tf.distribute.experimental.MultiWorkerMirroredStrategy() with mirrored_strategy.scope(): seq_data = keras.Input(shape=(None, 1), name="seq_data") seq_lengths = keras.Input(shape=(), name="seq_lengths", dtype=tf.int32) mask = keras.layers.Lambda(lambda x: tf.sequence_mask(x))(seq_lengths) conv = layers.Conv1D(32, 3, strides=1, padding='same', activation='relu')(seq_data) rnn = layers.Bidirectional(layers.GRU(32, return_sequences=True))(conv,mask=mask) #rnn = layers.Bidirectional(layers.GRU(32, return_sequences=True))(conv) dense = layers.Dense(5, name="signal_mask")(rnn) model = keras.Model(inputs=[seq_data, seq_lengths], outputs=[dense]) model.compile(optimizer=tf.keras.optimizers.Adam( learning_rate=0.001), loss=tf.keras.backend.sparse_categorical_crossentropy ) batch_size = 1 time_step = 1000 signal_length = 1000 sequences = np.random.rand(batch_size, time_step, 1) seq_lengths = np.array([signal_length]) signal_mask = np.random.randint(5, size=(1, time_step)) dataset = tf.data.Dataset.from_tensor_slices( (sequences, seq_lengths, signal_mask)).repeat(1000).batch(1)\ .map(lambda a,b,c: ({"seq_data":a,"seq_lengths":b}, {"signal_mask":c})) model.fit(dataset, epochs=1)
def create_stitchGAN_model(self,shape_in,shape_out,mode='stitch', model_path='',stateful=False,batch_size=1, optimizer=tf.keras.optimizers.Adam()): if stateful: print('\n Creating Stateful LSTM in inference Mode. \n') inputs = tf.keras.Input(batch_shape=[batch_size]+ list(shape_in[0][1:])) else: inputs = tf.keras.Input(shape=shape_in[0][1:]) lstm_out=layers.GRU(64,return_sequences=True, stateful=stateful)(inputs) #out= layers.Conv1D(8,1)(lstm_out) #out=layers.BatchNormalization()(lstm_out) #out=tf.keras.activations.relu(out) out=layers.Conv1D(1,1)(lstm_out) #inputs = tf.keras.Input(shape=shape_in[0][1:]) #gru_out=layers.GRU(64,return_sequences=True)(inputs) #gru_out=layers.Dropout(0.3,noise_shape=(None, 1, 64))(gru_out) #out= layers.Conv1D(1,1)(gru_out) model= tf.keras.Model(inputs=inputs,outputs=out) loss= tf.keras.losses.MeanSquaredError() metrics= ['mse'] model.compile(loss=loss, optimizer=optimizer, metrics=metrics) model.summary() print(model.optimizer) #tf.keras.utils.plot_model(model, "simple_stitcher.png") return model
def __init__(self, vocab_size: int, embedding_dim: int, filters: List[int], out_channels: int, drop_prob: float, nn_hidden_dim: int, gru_hidden_dim: 128, num_classes: int = 2): super(CNNandAttentiveBiGRUmodel, self).__init__() self._baseCNN = baseCNNmodel(embedding_dim, filters, out_channels) self.embedding_dim = embedding_dim self._embeddings = tf.Variable(tf.random.normal( (vocab_size, embedding_dim)), trainable=True) self.omegas = tf.Variable(tf.random.normal((gru_hidden_dim * 2, 1))) gru = layers.GRU(units=gru_hidden_dim, return_sequences=True) self.bi_gru_layer = layers.Bidirectional(layer=gru, merge_mode='concat') self.mlp1_layer = layers.Dense(units=nn_hidden_dim, activation='tanh') self.mlp2_layer = layers.Dense(units=50, activation='tanh') self._classification_layer = layers.Dense(units=num_classes) self.dropout = layers.Dropout(drop_prob)
def __init__(self, n_actions, input_shape): self.n_actions = n_actions self.input_shape = input_shape self._layers = [] if len(input_shape) > 1: self._layers.append( layers.Conv2D(filters=16, kernel_size=8, strides=4, activation=tf.nn.tanh)) self._layers.append( layers.Conv2D(filters=32, kernel_size=4, strides=2, activation=tf.nn.tanh)) self._layers.append(layers.Reshape((1, -1))) self._layers.append(layers.GRU(256, return_state=True)) else: self._layers.append(layers.Dense(units=32, activation=tf.nn.tanh)) self._layers.append(layers.Dense(units=32, activation=tf.nn.tanh)) # self._layers.append(layers.Reshape((1, -1))) # self._layers.append(layers.GRU(256, return_state = True)) # self._layers.append(layers.GRU(512, return_state = True, return_sequences = True)) self._actor = layers.Dense(self.n_actions, activation=tf.nn.softmax) self._critic = layers.Dense(1, activation=None)
def recurrent_model(units=32, celltype='GRU', Tx=None, Trej=0, print_summary=False, learning_rate=.001, name='recurrent_model', loss='mse', metrics='mse', initializer = keras.initializers.GlorotNormal()): ''' Recurrent neural network model. ''' # specify recurrent cell type if celltype=='GRU': Rcell = layers.GRU(units=units, return_sequences=True, kernel_initializer=initializer, name='Xrec') elif celltype=='LSTM': Rcell = layers.LSTM(units=units, return_sequences=True, kernel_initializer=initializer, name='Xrec') elif celltype=='RNN': Rcell = layers.SimpleRNN(units=units, return_sequences=True, kernel_initializer=initializer, name='Xrec') # Model architecture X_input = layers.Input(shape=(Tx,1), name='X0') X = Rcell(X_input) X = layers.Dense(1, activation='linear', kernel_initializer=initializer, name='Xdense')(X) Y = layers.Add(name='Y')([X,X_input]) # Create model model = keras.Model(inputs=[X_input], outputs=Y, name=name) if print_summary: model.summary() # Compile model opt = tf.keras.optimizers.Adam(learning_rate,clipvalue=10) model.compile(optimizer=opt, loss=loss, metrics=metrics) return model
def __init__(self, filters, kernel_size, num_residual_blocks, l2=None, **kwargs): super(StateEncoder, self).__init__(**kwargs) self.residual_blocks = [ ResidualBlock(filters=filters, kernel_size=3, l2=l2, name="residual_block_{:d}".format(i)) for i in range(num_residual_blocks) ] self.recurrency = layers.Bidirectional( layers.GRU( units=filters, stateful=False, return_sequences=True, return_state=True, implementation=2, # GPU compatible kernel_regularizer=regularizers.l2(l2)), name="recurrency") self.concatenate = layers.Concatenate() self.attention = BahdanauAttention(filters)
def build_model(self): """ builds and returns encoder decoder tensorflow model""" inputs = tf.keras.Input(shape=(None, )) x = layers.Embedding(self.num_encoder_tokens, self.latent_dim)(inputs) x = layers.Dropout(0.4)(x) if self.args.use_GRU: x = layers.GRU(64, return_sequences=False, return_state=False)(x) else: x = layers.LSTM(64, return_sequences=False, return_state=False)(x) x = layers.BatchNormalization()(x) x = layers.Dropout(0.4)(x) if self.args.use_document_context: verfahrensart_input = tf.keras.Input(shape=(1, )) dokumentart_input = tf.keras.Input(shape=(1, )) concat_list = [x, verfahrensart_input, dokumentart_input] x = tf.keras.layers.Concatenate()(concat_list) x = tf.keras.layers.Dense(255, activation="relu")(x) x = layers.Dropout(0.4)(x) outputs = layers.Dense(self.num_decoder_tokens, activation="softmax")(x) inputs_list = [inputs] if self.args.use_document_context: inputs_list.append(verfahrensart_input) inputs_list.append(dokumentart_input) model = Model(inputs_list, outputs) tf.keras.utils.plot_model(model, "RNN_model.png", show_shapes=True, show_layer_names=False) return model
def __init__( self, hidden_units, dropout_rate=0.2, aggregation_type="mean", combination_type="concat", normalize=False, *args, **kwargs, ): super(GraphConvLayer, self).__init__(*args, **kwargs) self.aggregation_type = aggregation_type self.combination_type = combination_type self.normalize = normalize self.ffn_prepare = create_ffn(hidden_units, dropout_rate) if self.combination_type == "gated": self.update_fn = layers.GRU( units=hidden_units, activation="tanh", recurrent_activation="sigmoid", dropout=dropout_rate, return_state=True, recurrent_dropout=dropout_rate, ) else: self.update_fn = create_ffn(hidden_units, dropout_rate)
def build_model(wav_size=16384, chunk_size=256, chunk_hop=128, emb_size=32): N_WIN = (wav_size - chunk_size) // chunk_hop + 1 hann_win = tf.signal.hann_window(chunk_size) wav_input = tf.keras.Input(shape=(wav_size, )) encoder = tf.signal.frame(wav_input, chunk_size, chunk_hop, axis=1) encoder = tf.multiply(encoder, hann_win) encoder = tf.expand_dims(encoder, axis=-1) encoder = tfkl.TimeDistributed(tfkl.Conv1D(64, 8, padding='SAME'))(encoder) encoder = tfkl.TimeDistributed( tfkl.Conv1D(64, 8, padding='SAME', strides=8))(encoder) encoder = tfkl.TimeDistributed( tfkl.Conv1D(64, 8, padding='SAME', strides=4))(encoder) encoder = tfkl.TimeDistributed( tfkl.Conv1D(64, 8, padding='SAME', strides=2))(encoder) encoder = tfkl.TimeDistributed(tfkl.Flatten())(encoder) encoder = tfkl.GRU(emb_size, return_sequences=True)(encoder) #Global LSTM encoder_model = tf.keras.Model(wav_input, encoder) decoder_input = tf.keras.Input(shape=(N_WIN, emb_size), name='encoded_img') decoder = tfkl.GRU(256, return_sequences=True)(decoder_input) decoder = tfkl.TimeDistributed(tfkl.Reshape(target_shape=(4, 64)))(decoder) decoder = tfkl.TimeDistributed(tfkl.Conv1D(64, 8, padding='SAME'))(decoder) decoder = tfkl.TimeDistributed(tfkl.UpSampling1D(2))(decoder) decoder = tfkl.TimeDistributed(tfkl.Conv1D(64, 8, padding='SAME'))(decoder) decoder = tfkl.TimeDistributed(tfkl.UpSampling1D(4))(decoder) decoder = tfkl.TimeDistributed(tfkl.Conv1D(64, 8, padding='SAME'))(decoder) decoder = tfkl.TimeDistributed(tfkl.UpSampling1D(8))(decoder) decoder = tfkl.TimeDistributed( tfkl.Conv1D(1, 8, padding='SAME', activation='tanh', dtype=tf.float32))(decoder) decoder = tfkl.Reshape(target_shape=(N_WIN, chunk_size))(decoder) decoder = tf.multiply(decoder, hann_win) decoder = tf.signal.overlap_and_add(decoder, chunk_hop) decoder_model = tf.keras.Model(decoder_input, decoder) ae_input = tf.keras.Input(shape=(wav_size, )) encoder_out = encoder_model(ae_input) decoder_out = decoder_model(encoder_out) ae_model = tf.keras.Model(inputs=ae_input, outputs=decoder_out) return ae_model, encoder_model, decoder_model
def main() -> None: data, res = gen_data_from_sequence() dataset_size = len(data) train_size = (dataset_size // 10) * 7 val_size = (dataset_size - train_size) // 2 train_data, train_res = data[:train_size], res[:train_size] val_data, val_res = data[train_size:train_size + val_size], res[train_size:train_size + val_size] test_data, test_res = data[train_size + val_size:], res[train_size + val_size:] model = Sequential() model.add( layers.GRU(128, recurrent_activation='sigmoid', input_shape=(None, 1), return_sequences=True)) model.add( layers.LSTM(64, activation='relu', input_shape=(None, 1), return_sequences=True, dropout=0.2)) model.add(layers.GRU(32, input_shape=(None, 1), recurrent_dropout=0.2)) model.add(layers.Dense(1)) model.compile(optimizer='nadam', loss='mse') history = model.fit(train_data, train_res, epochs=50, validation_data=(val_data, val_res)) loss = history.history['loss'] val_loss = history.history['val_loss'] print(np.mean(val_loss)) plt.plot(range(len(loss)), loss) plt.plot(range(len(val_loss)), val_loss) plt.show() predicted_res = model.predict(test_data) pred_length = range(len(predicted_res)) plt.plot(pred_length, predicted_res, "r") plt.plot(pred_length, test_res, "b") plt.legend() plt.show()
def build_model(input_dim, output_dim, rnn_layers=5, rnn_units=128): """Model similar to DeepSpeech2.""" # Model's input input_spectrogram = layers.Input((None, input_dim), name="input") # Expand the dimension to use 2D CNN. x = layers.Reshape((-1, input_dim, 1), name="expand_dim")(input_spectrogram) # Convolution layer 1 x = layers.Conv2D( filters=32, kernel_size=[11, 41], strides=[2, 2], padding="same", use_bias=False, name="conv_1", )(x) x = layers.BatchNormalization(name="conv_1_bn")(x) x = layers.ReLU(name="conv_1_relu")(x) # Convolution layer 2 x = layers.Conv2D( filters=32, kernel_size=[11, 21], strides=[1, 2], padding="same", use_bias=False, name="conv_2", )(x) x = layers.BatchNormalization(name="conv_2_bn")(x) x = layers.ReLU(name="conv_2_relu")(x) # Reshape the resulted volume to feed the RNNs layers x = layers.Reshape((-1, x.shape[-2] * x.shape[-1]))(x) # RNN layers for i in range(1, rnn_layers + 1): recurrent = layers.GRU( units=rnn_units, activation="tanh", recurrent_activation="sigmoid", use_bias=True, return_sequences=True, reset_after=True, name=f"gru_{i}", ) x = layers.Bidirectional( recurrent, name=f"bidirectional_{i}", merge_mode="concat" )(x) if i < rnn_layers: x = layers.Dropout(rate=0.5)(x) # Dense layer x = layers.Dense(units=rnn_units * 2, name="dense_1")(x) x = layers.ReLU(name="dense_1_relu")(x) x = layers.Dropout(rate=0.5)(x) # Classification layer output = layers.Dense(units=output_dim + 1, activation="softmax")(x) # Model model = keras.Model(input_spectrogram, output, name="DeepSpeech_2") # Optimizer opt = keras.optimizers.Adam(learning_rate=1e-4) # Compile the model and return model.compile(optimizer=opt, loss=CTCLoss) return model
def get_line_model(): line_input = layers.Input(shape=(LINE_LEN, INPUT_DIM)) masking = layers.Masking(0)(line_input) bi_seq = layers.Bidirectional(layers.GRU(128), merge_mode='sum')(masking) bi_seq = layers.BatchNormalization()(bi_seq) bi_seq = layers.Activation('relu')(bi_seq) return line_input, bi_seq