def build_nlp_model(q_input,rnn_dim,bidirection,dropout,lang_att): if not lang_att: q_input = Masking()(q_input) if bidirection: bi_rnn=gru_rnn_module_a(q_input,rnn_dim,dropout,lang_att) else: bi_rnn = gru_rnn_module_s(q_input, rnn_dim, dropout,lang_att) # bi_rnn = q_input if lang_att: bi_rnn_weights = Dense(K.int_shape(bi_rnn)[-1], activation='tanh')(bi_rnn) bi_rnn_weights = Dropout(0.1)(bi_rnn_weights) bi_rnn_weights = Lambda(K.softmax, arguments={'axis': 1})(bi_rnn_weights) bi_rnn = Multiply()([bi_rnn, bi_rnn_weights]) bi_rnn = Lambda(K.sum, arguments={'axis': 1})(bi_rnn) return bi_rnn
def build(self): word2vec_embedding_dim = self.config["params"]["word2vec"][ "embedding_dim"] embedding_dim = self.config["params"]["sentence_representation"][ "embedding_dim"] max_sentence_length = self.config["params"]["sentence_representation"][ "max_sentence_length"] pad_char_index = self.config["params"]["sentence_representation"][ "pad_char_index"] learning_rate = self.config["hyper_params"]["sentence_representation"][ "lr"] word_encoder_model, word_decoder_model = self.get_embedding_model() for layer in word_encoder_model.layers: layer.trainable = False for layer in word_decoder_model.layers: layer.trainable = False inputs = Input(shape=(max_sentence_length, 1)) mask = Masking(pad_char_index)(inputs) word_embedding = TimeDistributed(word_encoder_model)(mask) encoder = LSTM(32, return_sequences=True)(word_embedding) encoder = LSTM(64, return_sequences=True)(encoder) encoder = LSTM(128, return_sequences=True)(encoder) encoded = LSTM(embedding_dim, return_sequences=False)(encoder) repeated = RepeatVector(max_sentence_length)(encoded) decoder = LSTM(128, return_sequences=True)(repeated) decoder = LSTM(256, return_sequences=True)(decoder) decoder = TimeDistributed(Dense(word2vec_embedding_dim))(decoder) outputs = TimeDistributed(word_decoder_model)(decoder) model = Model(inputs=inputs, outputs=outputs) self.model = model self.model.summary() self.model.compile( loss="sparse_categorical_crossentropy", optimizer=Adam(learning_rate), metrics=["accuracy"], )
def make_rnn(vocab_size, embedding_size, max_len): input_layer = Input(shape=(None, ), name="input") mask = Masking(mask_value=0, name="mask")(input_layer) # embedding embedding_layer = Embedding(vocab_size, embedding_size, name="embedding")(mask) # rnn rnn_fn = LSTM # GRU/LSTM rnn_layers = [] lstms = [64] for i, count in enumerate(lstms): prev = embedding_layer if i == 0 else rnn_layers[-1] last = i == len(lstms) - 1 rnn_layers.append( rnn_fn( count, activation="relu", return_sequences=not last, name=f"recurrent{i+1}", )(prev)) # dense dense_layers = [] denses = [(128, 0.25), (64, 0.25)] for i, param in enumerate(denses): prev = rnn_layers[-1] if i == 0 else dense_layers[-1] dense_layers.append(Dense(param[0], name=f"dense{i+1}")(prev)) if param[1] > 0: dense_layers.append( Dropout(param[1], name=f"dropout{i+1}")(dense_layers[-1])) # out output_layer = Dense(1, activation="sigmoid", name="output")(dense_layers[-1]) model = Model(inputs=[input_layer], outputs=[output_layer]) model.compile( loss="binary_crossentropy", optimizer="nadam", metrics=["binary_accuracy"], ) return model
def make_model(self): with k.name_scope("SVM_features"): svm_features = Input(shape = (self.svm_dims,), name = "svm_features") svm_input = Dense(128, activation = "tanh", name = "svm_dense")(svm_features) # svm_input = LeakyReLU()(svm_input) with k.name_scope("LSTM_features"): lstm_features = Input(shape = (None, self.input_shape), name = "lstm_features") lstm_mask = Masking(mask_value = Config.MASKING_VALUE, input_shape = (self.time_steps, self.input_shape))(lstm_features) lstm_output, state_h, state_c = LSTM(Config.LSTM_UNITS, return_sequences = True, return_state = True, name = "lstm_sequence")(lstm_mask) # lstm_output_last = LSTM(Config.LSTM_UNITS, return_sequences = False, name = "lstm_last_output")(lstm_mask) with k.name_scope("AttentionLayer_1"): __, lstm_output_ex_last = Lambda(lambda t: [t, t[:, :-1, :]], name = "lstm_T1_Tn-1")(lstm_output) lstm_output_last = state_h attention_weights1 = dot([lstm_output_last, lstm_output_ex_last], name = "attention_weights1", axes = -1) # [B, 1, M] attention_weights2 = Activation("softmax", name = "attention_weights2")(attention_weights1) lstm_attention = dot([attention_weights2, lstm_output_ex_last], name = "lstm_attention", axes = 1) # final_attention = concatenate([lstm_attention, lstm_output_last]) print(lstm_attention) """ with k.name_scope("AttentionLayer_2"): # Attention layer 2 - attention params input_attention = Input(shape = (Config.ATTENTION_UNITS, ), name = "attention_params") u = Dense(Config.ATTENTION_UNITS, activation = "softmax", name = "attention_u")(input_attention) alpha = dot([u, lstm_output], axes = -1) alpha = Activation("softmax", name = "attention_weights")(alpha) # weighted pool lstm_attention = dot([alpha, lstm_output], name = "attention_output", axes = 1) """ with k.name_scope("Concatenate"): x = concatenate([lstm_attention, svm_input]) x_dense = Dense(128, activation = "tanh")(x) # x_dense = LeakyReLU()(x_dense) dense_2 = Dense(128, activation = "tanh")(x_dense) batchnorm2 = BatchNormalization()(dense_2) dropout = Dropout(rate = 0.3, name = "dropout")(batchnorm2) pred = Dense(self.num_classes, activation = "softmax", name = "output")(dropout) self.model = Model(inputs = [svm_features, lstm_features], outputs = [pred]) return self.model
def attention_model(lr=1e-5): predict_info = Input(shape=(None, 13), name='predict_info') sequence_data = Input(shape=(None, 14), name='sequence_data') item_info = Input(shape=(3,), name='item_info') x = Masking(mask_value=0, input_shape=(None, 19))(sequence_data) x = GRU(128, name='GRU_layer1', return_sequences=True)(x) # past sequence understanding attention_x = attention_mechanism(TRAIN_TIMESEQUENCE, x) gru_out = Permute((2, 1))(x) attention_mul = K.batch_dot(gru_out, attention_x) attention_mul = Permute((2, 1))(attention_mul) x = GRU(128, name='GRU_layer2', return_sequences=True)(attention_mul) # item feature added v = RepeatVector(TRAIN_TIMESEQUENCE)(item_info) # past sequence x = Concatenate(axis=-1)([v, x]) x = TimeDistributed(Dense(32, activation='relu'))(x) x = TimeDistributed(Dense(128, activation='relu'))(x) # get extraction of item feature x = Flatten()(x) v = RepeatVector(PREDICT_TIMESEQUENCE)(x) # predict sequence x = Concatenate(axis=-1)([predict_info, v]) x = GRU(128, name='GRU_predict_layer1', return_sequences=True)(x) # past sequence understanding attention_x = attention_mechanism(PREDICT_TIMESEQUENCE, x) gru_out = Permute((2, 1))(x) attention_mul = K.batch_dot(gru_out, attention_x) attention_mul = Permute((2, 1))(attention_mul) x = GRU(128, name='GRU_predict_layer2', return_sequences=True)(attention_mul) x = TimeDistributed(Dense(32, activation='relu'))(x) x = TimeDistributed(Dense(128, activation='relu'))(x) outputs = TimeDistributed(Dense(1, activation='sigmoid'))(x) print(outputs.shape) optimizer = Adam(lr=lr, name='adam') model = Model([sequence_data, item_info, predict_info], outputs, name='gru_network') model.compile(optimizer=optimizer, loss='mse') return model
def version_7(x_shape): model = Sequential() model.add(Masking(mask_value=0.,input_shape=(x_shape[1],x_shape[2]))) model.add(Bidirectional(LSTM(128, return_sequences=True))) model.add(Bidirectional(LSTM(64, return_sequences=True))) model.add(TimeDistributed(Dense(32, activation=None))) model.add(BatchNormalization(axis=-1, momentum=0.99)) model.add(Activation('relu')) model.add(TimeDistributed(Dense(1, activation=None))) model.add(BatchNormalization(axis=-1, momentum=0.99)) model.add(Activation('sigmoid')) model.compile(loss='binary_crossentropy', optimizer='adam', metrics=[rec_acc] ) return model
def main(selected_emotion): x_train, y_train, x_dev, y_dev, x_test, y_test = load_dataset(selected_emotion) x_train = np.vstack([x_train, x_dev]) # add dev data to train data y_train = np.append(y_train, y_dev, axis=0) # add dev labels to train labels new_x_train, new_y_train = subsampling(x_train, y_train) verbose, epochs, batch_size = 0, 15, 64 n_timesteps, n_features, n_outputs = new_x_train.shape[1], new_x_train.shape[2], 1 model = Sequential() model.add(Masking(mask_value=-1, input_shape=(n_timesteps, n_features))) model.add(LSTM(100, dropout=0.5)) model.add(Dense(100, activation='relu')) model.add(Dropout(0.5)) model.add(Dense(n_outputs, activation='sigmoid')) model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy']) # fit network model.fit(new_x_train, new_y_train, epochs=epochs, batch_size=batch_size, verbose=verbose) # Evaluate y_pred = model.predict(x_test) y_pred_class = np.where(y_pred>0.5, 1, 0) out_dir = os.path.join('results', 'LSTM', 'delaunay', selected_emotion) os.makedirs(out_dir, exist_ok=True) data = {'cohen_kappa': cohen_kappa_score(y_test, y_pred_class, weights='linear')} with open(os.path.join(out_dir, 'cohen_kappa.json'), 'w') as f: json.dump(data, f) clf_report = classification_report( y_test, y_pred_class, labels=[0, 1], output_dict=True) pd.DataFrame(clf_report).T.to_csv(os.path.join(out_dir, 'classification_report.csv')) sns.heatmap(pd.DataFrame(clf_report).iloc[:-1, :].T, annot=True) plt.savefig(os.path.join(out_dir, 'classification_report.png')) plt.close() confusion_mtx = confusion_matrix(y_test, y_pred_class, normalize='true') sns.heatmap(confusion_mtx, annot=True, fmt='.2g', cmap='Blues') plt.xlabel('Predicted label') plt.ylabel('True label') plt.savefig(os.path.join(out_dir, 'confusion_matrix.png'))
def create_discrete_encoder(enc_in_dim, layer_size=2048, latent_dim=1024, **kwargs): # Input # inputs = Input(shape=(None, enc_in_dim), dtype=tf.float32, name='encoder_in') # Layers # x = Masking(mask_value=0.)(inputs) x = Bidirectional(LSTM(layer_size, return_sequences=True), merge_mode='concat')(x) x = Bidirectional(LSTM(layer_size, return_sequences=False), merge_mode='concat')(x) logits = Dense(latent_dim, name='to_vocab')(x) return Model([inputs], logits)
def __init__(self, lstm_units=256, masking_enabled=True): super(se_comp_BiLSTM, self).__init__() self.masking_enabled = masking_enabled if masking_enabled: self.masking = Masking(mask_value=0.0) self.lstm_0 = LSTM( units=lstm_units, activation='tanh', recurrent_activation='sigmoid', # check this return_sequences=True, stateful=True, # return_state=True, name="lstm_0", # dropout=0.2, # recurrent_dropout=0.2, ) self.bilstm_0 = tf.keras.layers.Bidirectional(self.lstm_0, merge_mode='sum')
def get_audio_lstm(self): self.embedding_dim = self.train_x.shape[2] print("Creating Model...") inputs = Input(shape=(self.sequence_length, self.embedding_dim), dtype='float32') masked = Masking(mask_value =0)(inputs) lstm = Bidirectional(LSTM(200, activation='tanh', return_sequences = True, dropout=0.4), name='lstm_a')(masked) self.audio_lstm_layer = lstm lstm = Bidirectional(LSTM(200, activation='tanh', return_sequences = True, dropout=0.4), name="utter_a")(lstm) output = TimeDistributed(Dense(self.classes,activation='softmax',kernel_initializer='uniform'))(lstm) model = Model(inputs, output) self.audio_lstm = model return lstm, inputs
def create_encoder(obs_dim, act_dim, layer_size=2048, latent_dim=256): # Input # obs = Input(shape=(None, obs_dim), dtype=tf.float32, name='obs') acts = Input(shape=(None, act_dim), dtype=tf.float32, name='acts') # Layers # x = Concatenate(axis=-1)([obs, acts]) x = Masking(mask_value=0.)(x) x = Bidirectional(LSTM(layer_size, return_sequences=True), merge_mode='concat')(x) x = Bidirectional(LSTM(layer_size, return_sequences=False), merge_mode='concat')(x) # Latent Variable # x = Dense(tfpl.MultivariateNormalTriL.params_size(latent_dim), activation=None)(x), z = tfpl.MultivariateNormalTriL(latent_dim, name='latent')(x) return Model([obs, acts], z)
def create_discrete_encoder(obs_dim, act_dim, layer_size=2048, latent_dim=1024, **kwargs): # Input # obs = Input(shape=(None, obs_dim), dtype=tf.float32, name='obs') acts = Input(shape=(None, act_dim), dtype=tf.float32, name='acts') # Layers # x = Concatenate(axis=-1)([obs, acts]) x = Masking(mask_value=0.)(x) x = Bidirectional(LSTM(layer_size, return_sequences=True), merge_mode='concat')(x) x = Bidirectional(LSTM(layer_size, return_sequences=False), merge_mode='concat')(x) logits = Dense(latent_dim, name='to_vocab')(x) return Model([obs, acts], logits)
def create_model(n_labels: int, batch_size: int) -> object: nn_size = 300 embed_dim = 300 #print(bpemb_model.vectors.shape) wm = np.zeros((1000, 300)) for i in range(wm.shape[0]): # set pad to 0 if i == 999: wm[0] = bpemb_model.vectors[-1] else: wm[i + 1] = bpemb_model.vectors[i] model = Sequential() model.add( Embedding(1000, embed_dim, embeddings_initializer=Constant(wm), batch_input_shape=(batch_size, None))) model.add(Masking(mask_value=0, input_shape=(batch_size, None))) #model.add(Input(shape=(bs, None, embed_dim))) model.add( Bidirectional( LSTM(nn_size, return_sequences=True, stateful=False, dropout=0.3))) model.add( Bidirectional( LSTM(nn_size, return_sequences=True, stateful=False, dropout=0.3))) #model.add(Dense(n_labels-1, activation='softmax')) model.add(Dense(n_labels - 1, activation='sigmoid')) #model.compile(loss='binary_crossentropy',#'mean_squared_error', model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy']) print(model.summary()) return model
def get_model(self): input_text = Input(shape=(self.max_len_question, self.latent_dim)) mask = Masking(mask_value=np.zeros(self.latent_dim))(input_text) """ # lstm lstm = LSTM(self.latent_dim, return_sequences=False, return_state=False) lstm_output = lstm(input_vec) layer_norm1 = LayerNormalization(axis=1)(lstm_output) # dense dense = Dense(1, activation='sigmoid') dense_output = dense(layer_norm1) """ fe = Conv1D(16, 3, strides=2, padding='same')(mask) fe = LeakyReLU(alpha=0.2)(fe) fe = Dropout(0.2)(fe) # normal fe = Conv1D(32, 3, strides=2, padding='same')(fe) fe = BatchNormalization()(fe) fe = LeakyReLU(alpha=0.2)(fe) fe = Dropout(0.2)(fe) # downsample to 7x7 fe = Conv1D(64, 3, strides=2, padding='same')(fe) fe = BatchNormalization()(fe) fe = LeakyReLU(alpha=0.2)(fe) fe = Dropout(0.2)(fe) # downsample one more fe = Conv1D(128, 3, strides=2, padding='same')(fe) fe = BatchNormalization()(fe) fe = LeakyReLU(alpha=0.2)(fe) fe = Dropout(0.2)(fe) # flatten feature maps fe = Flatten()(fe) # real/fake output out1 = Dense(1, activation='sigmoid')(fe) model = Model(input_text, out1) return model
def __init__(self, output_vec_len=300, masking_enabled=True, SeATT_enabled=True, **kwargs): super(WE_SeAtt_BiLSTM, self).__init__(**kwargs) self.masking_enabled = masking_enabled if masking_enabled: self.masking = Masking(mask_value=0.0) self.dense_0 = Dense(units=300) self.lstm_0 = LSTM( units=output_vec_len, activation='tanh', recurrent_activation='sigmoid', # check this return_sequences=True, stateful=True, return_state=True, name="lstm_0", # dropout=0.2, # recurrent_dropout=0.2, ) self.bilstm_0 = tf.keras.layers.Bidirectional(self.lstm_0, merge_mode='sum') self.lstm_1 = LSTM( units=output_vec_len, activation='tanh', recurrent_activation='sigmoid', # check this return_sequences=True, stateful=True, return_state=False, name="lstm_1", # dropout=0.2, # recurrent_dropout=0.2, ) self.dropout_0 = tf.keras.layers.Dropout(0.2) self.bilstm_1 = tf.keras.layers.Bidirectional(self.lstm_1, merge_mode='sum') if SeATT_enabled: self.sat = custom_SAT() else: self.max_pool = MaxPool2D
def create_model(self, model: Optional[IModel]) -> IModel: assert model is None # None here means variable over batches (but not within a batch) input = Input(shape=(self.params.max_timesteps, self.params.swipe_feature_count)) masking = Masking(mask_value=myNaN)(input) d = Dense(50, kernel_initializer='random_uniform', activation='linear')(masking) lstm = LSTM(16, kernel_initializer='random_uniform')(d) middle = Dense(50, kernel_initializer='random_uniform', activation='relu')(lstm) output = Dense(1, kernel_initializer='random_uniform', activation='sigmoid')(middle) model = Model(inputs=[input], outputs=output) return model
def blstm_architecture(input_shape, output_shape, rnn_size=100, dropout=0, dense_units=100, learning_rate=0.01, **kwargs): """ The architecture of a bidirectional lstm model. (Dense Layer, forward LSTM Layer, backward LSTM Layer, Dense Output Layer) Parameters: input_shape (tuple): contains part of the input shape of the data, namely the maximal length of time points and the number of features output_shape (int): contains part of the output shape of the data, namely the number of labels rnn_size (int): how many units the LSTM layers should have dropout (float): how many percent of the units in the layers should drop out dense_units (int): how many units the first dense layer should have. Default=100. learning_rate (float): which learning rate the Adam optimizer should use. Default=0.01. Return: model: a compiled model ready for fitting """ # architecture # change this: https://stackoverflow.com/questions/62991082/bidirectional-lstm-merge-mode-explanation model = Sequential() model.add(Masking(mask_value=0.0, input_shape=(None, input_shape[1]))) if dense_units > 0: model.add(Dense(units=dense_units, activation="relu")) model.add(Dropout(dropout)) forward_layer = LSTM(rnn_size, return_sequences=True) backward_layer = LSTM(rnn_size, return_sequences=True, go_backwards=True) model.add( Bidirectional(forward_layer, backward_layer=backward_layer, merge_mode="concat")) model.add(Dropout(dropout)) model.add(Dense(units=output_shape, activation="relu")) model.add(Activation("softmax")) optimizer = keras.optimizers.Adam(learning_rate=learning_rate) model.compile(optimizer=optimizer, loss="categorical_crossentropy", metrics=["acc"]) return model
def create_actor(obs_dim, act_dim, goal_dim, layer_size=1024, latent_dim=256, training=True): # params # batch_size = None if training else 1 stateful = not training # Input # o = Input(shape=(None, obs_dim), batch_size=batch_size, dtype=tf.float32, name='input_obs') z = Input(shape=(None, latent_dim), batch_size=batch_size, dtype=tf.float32, name='input_latent') g = Input(shape=(None, goal_dim), batch_size=batch_size, dtype=tf.float32, name='input_goals') # RNN # x = Concatenate(axis=-1)([o, z, g]) x = Masking(mask_value=0.)(x) x = LSTM(layer_size, return_sequences=True, stateful=stateful, name='LSTM_in_1')(x) x = LSTM(layer_size, return_sequences=True, stateful=stateful, name='LSTM_in_2')(x) # Deterministic output # actions = Dense(act_dim, activation='tanh', name='acts')(x) actions = Lambda(lambda a: a * ACT_LIMITS)( actions) # scale to action limits return Model([o, z, g], actions)
def get_LSTM(timesteps, num_feature, num_class, n_hidden=100, inter_num=0, inter_nodes=10, drop_rate=0.5, mask_value=None, optimizer='Adam', learning_rate=1e-2, loss=None, final_activation=None): opt = None if optimizer == 'SGD': opt = SGD(lr=learning_rate) if optimizer == 'RMSprop': opt = RMSprop(lr=learning_rate) if optimizer == 'Adadelta': opt = Adadelta(lr=learning_rate) if optimizer == 'Adam': opt = Adam(lr=learning_rate) if loss == None: loss = ('binary_crossentropy' if num_class <= 2 else 'categorical_crossentropy') if final_activation == None: final_activation = ('sigmoid' if num_class <= 2 else 'softmax') inputs = Input(shape=(timesteps, num_feature)) x_nn = inputs if mask_value != None: x_nn = Masking(mask_value=mask_value)(x_nn) lstm_out = LSTM(n_hidden, return_sequences=True)(x_nn) inter = lstm_out for i in range(inter_num): inter = TimeDistributed(Dense(inter_nodes, activation='relu'))(inter) inter = TimeDistributed(Dropout(drop_rate))(inter) outputs = TimeDistributed(Dense(num_class, activation=final_activation))(inter) model = Model(inputs=inputs, outputs=outputs) model.compile(optimizer=optimizer, loss=loss) return model
def generate_model(self): # Input input = Input(shape=[None, self.args.num_mels], name='input_speech') x = Masking(mask_value=0, input_shape=(None, self.args.num_mels))(input) # Bidirectional LSTM for b in range(self.args.num_lstm_layers): x = Bidirectional( GRU(units=self.args.num_units_per_lstm // 2, return_sequences=True, recurrent_initializer='glorot_uniform'))(x) x = LayerNormalization(epsilon=1e-6)(x) # output layer with softmax logit = Dense(self.args.num_classes, name='speech_encoder', activation='softmax')(x) # Generate model self.model = Model(inputs=input, outputs=logit)
def __init__( self, num_hidden, embedding=False, embedding_len = 50 ): """ :param num_hidden: :param embedding: :param embedding_len: """ super( SequenceAttentionLayer, self ).__init__() self.num_hidden = num_hidden self.if_embedding = embedding self.embedding_len = embedding_len if embedding: self.embedding_len = embedding_len self.embed = Embedding( input_dim=len(word_vectors), output_dim=self.embedding_len, weights=[word_vectors], mask_zero=True, trainable=False ) self.embed_mask = Masking(mask_value=0) self.lstm = LSTM( self.num_hidden, activation="tanh", return_sequences=True ) # pass the type of attention self.attention_layer = AdditiveAttention( use_scale=True ) self.attention_layer.__setattr__('supports_masking', True)
def create_model(vocab: Dict[bytes, int], labels: list, batch_size: int) -> object: nn_size = 100 embed_dim = 300 model = Sequential() model.add( Embedding(10000, embed_dim, mask_zero=True, batch_input_shape=(batch_size, None))) model.add(Masking(mask_value=0., input_shape=(batch_size, None))) model.add( Bidirectional( LSTM(nn_size, return_sequences=True, stateful=False, dropout=0.3))) #model.add(Bidirectional(LSTM(nn_size, # return_sequences=True, # stateful=False, # dropout=0.3))) #model.add(Bidirectional(LSTM(nn_size, # return_sequences=True, # stateful=False, # dropout=0.3))) #model.add(LSTM(nn_size, return_sequences=True)) model.add(TimeDistributed(Dense(len(labels), activation='softmax'))) model.compile( loss='sparse_categorical_crossentropy', #'mean_squared_error', optimizer='adam', metrics=['sparse_categorical_accuracy']) print(model.summary()) return model
def my_model(input_shape): # model # Δημιουργία μοντέλου με χρήση του keras API model = Sequential() # Πρώτο κρυφό επίπεδο model.add(Dense(794, input_shape=(784, ), activation='relu')) model.add(Masking(mask_value=0.0, input_shape=input_shape)) # Δεύτερο κρυφό επίπεδο model.add(Dense(50, activation='relu')) # Επίπεδο εξόδου model.add(Dense(10, activation='softmax')) # compile the model model.load_weights('my_model_weights.h5') # model = tensorflow.keras.models.load_model('my_model.h5') model.compile(loss='categorical_crossentropy', optimizer=tensorflow.keras.optimizers.SGD(learning_rate=0.1, momentum=0.6, decay=0.0, nesterov=False), metrics=['accuracy']) return model
def get_model(self): input_text = Input(shape=(self.max_len_question, self.embed_size)) mask = Masking(mask_value=np.zeros(self.embed_size))(input_text) # lstm1 lstm1 = LSTM(256, return_sequences=True, return_state=True) lstm1_output, state_h1, state_c1 = lstm1(mask) layer_norm1 = LayerNormalization(axis=1)(lstm1_output) # lstm2 lstm2 = LSTM(128, return_sequences=True, return_state=True) lstm2_output, state_h2, state_c2 = lstm2(layer_norm1) layer_norm2 = LayerNormalization(axis=1)(lstm2_output) # lstm3 lstm3 = LSTM(64, return_sequences=True, return_state=True) lstm3_output, state_h3, state_c3 = lstm3(layer_norm2) layer_norm3 = LayerNormalization(axis=1)(lstm3_output) # lstm4 lstm4 = LSTM(128, return_sequences=True, return_state=True) lstm4_output, state_h4, state_c4 = lstm4(layer_norm3) layer_norm4 = LayerNormalization(axis=1)(lstm4_output) # lstm5 lstm5 = LSTM(256, return_sequences=True, return_state=True) lstm5_output, state_h5, state_c5 = lstm5(layer_norm4) layer_norm5 = LayerNormalization(axis=1)(lstm5_output) # lstm6 lstm6 = LSTM(self.embed_size, return_sequences=True, return_state=True, activation='tanh') lstm6_output, state_h6, state_c6 = lstm6(layer_norm5) model = Model(inputs=input_text, outputs=lstm6_output) return model
def call(self, inp): """ Compute attention mask. Argument/s: inp - used to compute sequence mask. Returns: Attention mask. """ batch_size = tf.shape(inp)[0] max_seq_len = tf.shape(inp)[1] flat_seq_mask = Masking(mask_value=0.0).compute_mask(inp) seq_mask = self.merge_masks(tf.expand_dims(flat_seq_mask, axis=1), tf.expand_dims(flat_seq_mask, axis=2)) causal_mask = self.lower_triangular_mask([1, max_seq_len, max_seq_len ]) if self.causal else None logical_mask = self.merge_masks(causal_mask, seq_mask) att_mask = tf.cast(logical_mask, tf.float32) att_mask = tf.reshape(att_mask, [batch_size, 1, max_seq_len, max_seq_len]) return att_mask
def attention_model(input_size, days=21, batch_size=32, epochs=200, lr=1e-3): country_input = Input(shape=(313, ), name='country_onehot') inputs = Input(shape=(None, input_size), name='encoder_input') target_number = Input(shape=(1, ), name='target_input') flag_input = Input(shape=(1, ), name='flag_input') x = Masking(mask_value=0, input_shape=(None, input_size))(inputs) x = GRU(128, name='GRU_layer1', return_sequences=True)(x) attention_x = attention_mechanism(days, x) gru_out = Permute((2, 1))(x) attention_mul = K.batch_dot(gru_out, attention_x) attention_mul = Permute((2, 1))(attention_mul) x = GRU(128, name='GRU_layer2', return_sequences=True)(attention_mul) gru_x = Flatten()(x) # country onehot concatenate # x = Concatenate(axis=-1)([country_input, gru_x]) x = Dense(32, activation='relu')(gru_x) # x = BatchNormalization()(x) x = Dense(128, activation='relu')(x) # x = BatchNormalization()(x) x = Dense(256, activation='relu')(x) # x = Dense(16, activation='relu')(x) # x = Concatenate(axis=-1)([x, gru_x]) outputs = Dense(1, activation='sigmoid')(x) outputs = target_number * (flag_input + outputs) print(outputs.shape, flag_input.shape, target_number.shape) optimizer = Adam(lr=lr, name='adam') model = Model([inputs, country_input, target_number, flag_input], outputs, name='gru_network') model.compile(optimizer=optimizer, loss=rmsle) # print(self.model.summary()) return model
def Encoder(): inputs = Input(shape=(None, ), name="encoder_inputs") masking = Masking(mask_value=0, name="Encoder_masking") embedding = Embedding(input_dim=hps.n_symbols, output_dim=hps.embedding_dim, name="Character_embedding") bidlstm = Bidirectional(LSTM(units=hps.lstm_size, recurrent_dropout=hps.lstm_drop_rate, return_sequences=True), name='Bidir_LSTM') conv1d_3 = Conv1d_3() x = masking(inputs) # making 대신 embedding masking zero 도 있는데 확인은 안해보았다. x = embedding(x) x = conv1d_3(x) outputs = bidlstm(x) return Model(inputs, x, name="encoder")
def version_8(x_shape): model = Sequential() model.add(Masking(mask_value=0., input_shape=(x_shape[1], x_shape[2]))) model.add( Bidirectional(LSTM(64, return_sequences=True), input_shape=(x_shape[1], x_shape[2]))) model.add(Bidirectional(LSTM(128))) model.add(Dense(64, activation=None)) model.add(BatchNormalization(axis=-1, momentum=0.99)) model.add(Activation('relu')) model.add(Dense(32, activation=None)) model.add(BatchNormalization(axis=-1, momentum=0.99)) model.add(Activation('relu')) model.add(Dense(1, activation=None)) model.add(BatchNormalization(axis=-1, momentum=0.99)) model.add(Activation('sigmoid')) model.compile('adam', 'binary_crossentropy', metrics=['accuracy']) return model
def seqvector_LSTM(obj): mirrored_strategy = MirroredStrategy() with mirrored_strategy.scope(): METRICS = [ keras.metrics.CategoricalAccuracy(name='accuracy'), keras.metrics.Precision(name='precision'), keras.metrics.Recall(name='recall'), keras.metrics.AUC(name='auc') ] model = keras.models.Sequential(name='seqvector_LSTM') model.add( Masking(mask_value=0., input_shape=obj.train_set[0].shape[1:])) model.add(Bidirectional(LSTM(units=128, dropout=.2))) model.add(Dense(units=128, activation='relu')) model.add(Dropout(.4)) model.add(Dense(units=64, activation='relu')) model.add(Dropout(.2)) model.add(Dense(units=21, activation='softmax', dtype='float32')) model.compile(optimizer=keras.optimizers.Adam(), loss='categorical_crossentropy', metrics=METRICS) model.summary() return model
def create_goal_space_mapper(input_embedding_dim, goal_embedding_dim, layer_size=2048, **kwargs): # params # batch_size = None # Input # input_embeddings = Input( shape=(input_embedding_dim, ), batch_size=batch_size, dtype=tf.float32, name='lang_embeds') # embeddings created by MUSE or equiv # Layers # x = Masking(mask_value=0.)(input_embeddings) x = Dense(layer_size, activation="relu", name='layer_1')(x) x = Dense(layer_size, activation="relu", name='layer_2')(x) goal_embeddings = Dense(goal_embedding_dim, activation=None, name='goal_space')(x) return Model(input_embeddings, goal_embeddings)