# print(train_images_mnist.shape) # print(test_images_mnist) print('test_images_mnist:', test_images_mnist.shape) print('test_labels_mnist:', test_labels_mnist.shape) sgd = optimizers.SGD(learning_rate=0.01, decay=1e-6, momentum=0.9, nesterov=True) model.compile( optimizer=sgd, loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True), metrics=['accuracy']) history = model.fit(train_images_mnist, train_labels_mnist, batch_size=256, epochs=200, validation_data=(test_images_mnist, test_labels_mnist)) plt.plot(history.history["accuracy"]) plt.plot(history.history['val_accuracy']) plt.plot(history.history['loss']) plt.plot(history.history['val_loss']) plt.title("model accuracy") plt.ylabel("Accuracy") plt.xlabel("Epoch") plt.legend(["Accuracy", "Validation Accuracy", "loss", "Validation Loss"]) plt.show()
def LSTM_model_veracity(x_train_embeddings, x_train_metafeatures, y_train, x_test_embeddings, x_test_metafeatures, params, eval=False, use_embeddings=True, use_metafeatures=True, Early_Stopping=True, log_path=""): # Parameter search log_dir = log_path + datetime.datetime.now().strftime("%d%m%Y-%H%M%S") num_lstm_units = int(params['num_lstm_units']) num_lstm_layers = int(params['num_lstm_layers']) num_dense_layers = int(params['num_dense_layers']) num_dense_units = int(params['num_dense_units']) num_epochs = params['num_epochs'] learn_rate = params['learn_rate'] mb_size = params['mb_size'] l2reg = params['l2reg'] dropout = params['dropout'] attention = params['attention'] # Defining input shapes if use_embeddings: emb_shape = x_train_embeddings[0].shape if use_metafeatures: metafeatures_shape = x_train_metafeatures[0].shape # Creating the two inputs if use_embeddings: emb_input = Input(shape=emb_shape, name='Embeddings') if use_metafeatures: metafeatures_input = Input(shape=metafeatures_shape, name='Metafeatures') # Adding masks to account for zero paddings if use_embeddings: emb_mask = Masking(mask_value=0, input_shape=(None, emb_shape))(emb_input) if use_metafeatures: metafeatures_mask = (Masking( mask_value=0, input_shape=(None, metafeatures_shape)))(metafeatures_input) # Adding attention and LSTM layers with varying layers and units using parameter search if attention == 1: for nl in range(num_lstm_layers): if use_embeddings: emb_LSTM_query = Bidirectional( LSTM(num_lstm_units, dropout=dropout, recurrent_dropout=0.2, return_sequences=True))(emb_mask) emb_LSTM_value = Bidirectional( LSTM(num_lstm_units, dropout=dropout, recurrent_dropout=0.2, return_sequences=True))(emb_mask) if use_metafeatures: metafeatures_LSTM_query = Bidirectional( LSTM(num_lstm_units, dropout=dropout, recurrent_dropout=0.2, return_sequences=True))(metafeatures_mask) metafeatures_LSTM_value = Bidirectional( LSTM(num_lstm_units, dropout=dropout, recurrent_dropout=0.2, return_sequences=True))(metafeatures_mask) if use_embeddings: emb_LSTM = AdditiveAttention(name='Attention_Embeddings')( [emb_LSTM_query, emb_LSTM_value]) if use_metafeatures: metafeatures_LSTM = AdditiveAttention( name='Attention_Metafeatures')( [metafeatures_LSTM_query, metafeatures_LSTM_value]) else: if use_embeddings: emb_LSTM = Bidirectional( LSTM(num_lstm_units, dropout=dropout, recurrent_dropout=dropout, return_sequences=True))(emb_mask) if use_metafeatures: metafeatures_LSTM = Bidirectional( LSTM(num_lstm_units, dropout=dropout, recurrent_dropout=dropout, return_sequences=True))(metafeatures_mask) if use_embeddings and use_metafeatures: # Concatenating the two inputs model = Concatenate()([emb_LSTM, metafeatures_LSTM]) elif use_metafeatures: model = metafeatures_LSTM # Adding attention and another LSTM to the concatenated layers if attention == 1: model_query = Bidirectional( LSTM(num_lstm_units, dropout=dropout, recurrent_dropout=0.2, return_sequences=False))(model) model_value = Bidirectional( LSTM(num_lstm_units, dropout=dropout, recurrent_dropout=0.2, return_sequences=False))(model) model = AdditiveAttention(name='Attention_Model')( [model_query, model_value]) else: model = Bidirectional( LSTM(num_lstm_units, dropout=dropout, recurrent_dropout=dropout, return_sequences=False))(model) # Adding dense layer with varying layers and units using parameter search for nl in range(num_dense_layers): model = Dense(num_dense_units)(model) model = LeakyReLU()(model) # Adding dropout to the model model = Dropout(dropout)(model) # Adding softmax dense layer with varying l2 regularizers using parameter search output = Dense(3, activation='softmax', activity_regularizer=regularizers.l2(l2reg), name='labels')(model) # Model output if use_embeddings and use_metafeatures: model = Model(inputs=[emb_input, metafeatures_input], outputs=output) elif use_metafeatures: model = Model(inputs=metafeatures_input, outputs=output) #model = Model(inputs=emb_input, outputs=output) # Plotting the model #plot_model(model, to_file='model_plot.png', show_shapes=True, show_layer_names=True) # Adding Adam optimizer with varying learning rate using parameter search adam = optimizers.Adam(lr=learn_rate, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=0.0) # Compiling model model.compile(optimizer=adam, loss='categorical_crossentropy', metrics=['accuracy']) callback_list = [] #TensorBoard tensorboard_callback = TensorBoard(log_dir=log_dir, histogram_freq=1) callback_list.append(tensorboard_callback) #Early_Stopping if Early_Stopping: earlystop_callback = EarlyStopping(monitor='val_accuracy', min_delta=0.0001, patience=5) callback_list.append(earlystop_callback) #plot_model(model, "model.png") if Early_Stopping: # Fitting the model with varying batch sizes and epochs using parameter search if use_embeddings and use_metafeatures: model.fit( { 'Embeddings': x_train_embeddings, 'Metafeatures': x_train_metafeatures }, y_train, batch_size=mb_size, epochs=num_epochs, shuffle=True, class_weight=None, verbose=1, callbacks=callback_list, validation_split=.1) elif use_metafeatures: model.fit(x_train_metafeatures, y_train, batch_size=mb_size, epochs=num_epochs, shuffle=True, class_weight=None, verbose=1, callbacks=callback_list, validation_split=.1) else: # Fitting the model with varying batch sizes and epochs using parameter search if use_embeddings and use_metafeatures: model.fit( { 'Embeddings': x_train_embeddings, 'Metafeatures': x_train_metafeatures }, y_train, batch_size=mb_size, epochs=num_epochs, shuffle=True, class_weight=None, verbose=1, callbacks=callback_list) elif use_metafeatures: model.fit(x_train_metafeatures, y_train, batch_size=mb_size, epochs=num_epochs, shuffle=True, class_weight=None, verbose=1, callbacks=callback_list) # Evaluation time if eval == True: model.save('output\\model_veracity.h5') json_string = model.to_json() with open('output\\model_architecture_veracity.json', 'w') as fout: json.dump(json_string, fout) model.save_weights('output\\model_veracity_weights.h5') # Getting confidence of the model if use_embeddings and use_metafeatures: pred_probabilities = model.predict( [x_test_embeddings, x_test_metafeatures], batch_size=mb_size, verbose=0) confidence = np.max(pred_probabilities, axis=1) # Getting predictions of the model y_prob = model.predict([x_test_embeddings, x_test_metafeatures], batch_size=mb_size) Y_pred = y_prob.argmax(axis=-1) elif use_metafeatures: pred_probabilities = model.predict(x_test_metafeatures, batch_size=mb_size, verbose=0) confidence = np.max(pred_probabilities, axis=1) # Getting predictions of the model y_prob = model.predict(x_test_metafeatures, batch_size=mb_size) Y_pred = y_prob.argmax(axis=-1) return Y_pred, confidence