def evaluate_model(model, data_reader, test_dir, batch_size): """Test the model on test dataset attained from test_dir. Args: model: the trained keras model data_reader: the TfRecords datareader used to load training and validation datasets test_dir: string representing the directory containing the test TfRecords files Returns: eval_dict: dictionary containing important evaulation metrics """ #Create evaluation metrics aucroc_calculator = metrics.AUC(multi_label=True) aucpr_calculator = metrics.AUC(multi_label=True, curve='PR') pr_calculator = metrics.PrecisionAtRecall(0.7) rp_calculator = metrics.RecallAtPrecision(0.7) #Prepare data batch_num = 0 test_dataset = data_reader.get_dataset(test_dir, batch_size=batch_size, type="validate") test_dataset = tfds.as_numpy(test_dataset) for batch in test_dataset: test_input = tf.convert_to_tensor(batch[0]) test_labels = tf.convert_to_tensor(batch[1]) predictions = model.predict(test_input) loss_val = loss.custom_crossentropy(test_labels, predictions) #Update Metrics aucroc_calculator.update_state(test_labels, predictions) aucpr_calculator.update_state(test_labels, predictions) pr_calculator.update_state(test_labels, predictions) rp_calculator.update_state(test_labels, predictions) print(f"Batch Number {batch_num} with loss {loss_val}.") batch_num += 1 #Get results auc_roc = aucroc_calculator.result() auc_pr = aucpr_calculator.result() precision = pr_calculator.result() recall = rp_calculator.result() eval_dict = { "AUCPR": auc_pr, "AUCROC": auc_roc, "precision": precision, "recall": recall } return eval_dict
def test_eval(model, test_df, y_test): test_data = BertPreprocessing( test_df[["sentence1", "sentence2"]].values.astype("str"), y_test, batch_size=config.batch_size, shuffle=False, ) y_pred = model.predict(test_data) size = y_pred.shape[0] y_test = y_test[:size, :] accuracy = metrics.CategoricalAccuracy() accuracy.update_state(y_test, y_pred) precision = metrics.Precision() precision.update_state(y_test, y_pred) recall = metrics.Recall() recall.update_state(y_test, y_pred) f1 = tfa.metrics.F1Score(num_classes=3, average="macro") f1.update_state(y_test, y_pred) auc = metrics.AUC() auc.update_state(y_test, y_pred) print(f""" Accuracy: {accuracy.result().numpy()} Precision: {precision.result().numpy()} Recall: {recall.result().numpy()} F1 score: {f1.result().numpy()} AUC: {auc.result().numpy()} """)
def eval_use_model(model_name, path_model_file, test_file, class_num): """ evaluating model by using entire model (weights, architecture, optimizers, etc.) Arguments:\n model_name --> String, Resnet50/Resnet18/VGG16/VGG19 path_model_file --> String, path which store .hdf5 of model's weight\n test_file --> String, path to which store .h5 file of test dataset class_num --> Int, number of class/label\n Returns:\n none """ # Load model weights new_model = Model() new_model = load_model(path_model_file) new_model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=[ metrics.AUC(), metrics.CategoricalAccuracy(), metrics.TruePositives(), metrics.TrueNegatives(), metrics.FalsePositives(), metrics.FalseNegatives() ]) # retrieve X_test, Y_test X_test, Y_test = retrieve_test_dataset(test_file, int(class_num)) for i in range(4): hasil = new_model.evaluate(X_test, Y_test) print(new_model.metrics_names) print(hasil)
def define_model(self, length, vocab_size, num_outcome_classes): ''' Defines and compiles a convolutional neural network model Parameters ___________ length: int Max length of the text strings vocab_size: int Vocabulary size of the text documents ''' input = Input(shape=(length, )) embed = Embedding(vocab_size, 200)(input) gru = Bidirectional(GRU(128, return_sequences=True))(embed) drop1 = SpatialDropout1D(0.2)(gru) conv = Conv1D(filters=256, kernel_size=4, activation='relu')(drop1) pool = MaxPooling1D(pool_size=2)(conv) att = Attention()([gru, pool]) flat = Flatten()(att) drop2 = Dropout(.2)(flat) dense1 = Dense(64, activation='relu')(drop2) output = Dense(num_outcome_classes, activation='softmax')(dense1) model = Model(inputs=input, outputs=output) model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['categorical_accuracy', metrics.AUC()]) self.model = model
def define_model(self, length, vocab_size): ''' Defines and compiles a convolutional neural network model Parameters ___________ length: int Max length of the text strings vocab_size: int Vocabulary size of the text documents ''' model = Sequential() model.add(Embedding(vocab_size, 125, input_length=length)) model.add(SpatialDropout1D(0.2)) model.add(GRU(64)) #model.add(Bidirectional(GRU(75))) model.add(Dropout(0.25)) model.add(Dense(3, activation='softmax')) model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy', metrics.AUC()]) #model = Model(inputs=inputs1, outputs=outputs) #model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy', metrics.AUC()]) self.model = model
def confusion_matrix_other_metric(self): return [ metrics.Accuracy(name='acc'), metrics.Precision(name='precision'), metrics.Recall(name='recall'), metrics.AUC(name='auc'), ]
def train_network(training_set, training_labels, save_path='network.keras'): global model print('----TRAINING----') model = Sequential([ Flatten(input_shape=training_set[0].shape), Dense(256, activation='relu'), Dense(64, activation='relu'), Dense(10, activation='softmax'), ]) model_metrics = [ metrics.CategoricalAccuracy(), metrics.Recall(), metrics.AUC(), metrics.SensitivityAtSpecificity(.8), metrics.SpecificityAtSensitivity(.8), f1_score, fbeta_score ] model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=model_metrics) print(model.summary()) train_time = time.time_ns() history = model.fit(training_set, training_labels, epochs=2, batch_size=32, validation_split=0) print((time.time_ns() - train_time) / 1000000) print(str(history.history['loss'])[1:-1].replace(',', '')) print(str(history.history['categorical_accuracy'])[1:-1].replace(',', '')) model.save(save_path)
def define_model(self, length, vocab_size, num_outcome_classes): ''' Defines and compiles a convolutional neural network model Parameters ___________ length: int Max length of the text strings vocab_size: int Vocabulary size of the text documents ''' model = Sequential() model.add(Input(shape=(length, ))) model.add(Embedding(vocab_size, 200)) model.add(Bidirectional(GRU(128, return_sequences=True))) model.add(SpatialDropout1D(0.2)) model.add(Conv1D(filters=64, kernel_size=4, activation='relu')) model.add(MaxPooling1D(pool_size=2)) model.add(Flatten()) model.add(Dropout(.2)) model.add(Dense(32, activation='relu')) model.add(Dense(num_outcome_classes, activation='softmax')) model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['categorical_accuracy', metrics.AUC()]) self.model = model
def get_metrics(): acc = 'accuracy' auc = metrics.AUC(num_thresholds=200, curve='ROC', name='auc', thresholds=None, multi_label=False) fp = metrics.FalsePositives(thresholds=[0.001, 0.01, 0.1, 1.0], name='FP') tp = metrics.TruePositives(thresholds=[0.001, 0.01, 0.1, 1.0], name='TP') return [acc] #, auc, fp, tp]
def train_din(mode=1): hidden_unit = 64 batch_size = 32 learning_rate = 1 epochs = 50 with open(electronics.pkl_dataset, 'rb') as f: train_set = np.array(pickle.load(f)) test_set = pickle.load(f) cate_list = pickle.load(f) user_count, item_count, cate_count, max_sl = pickle.load(f) train_user, train_item, train_hist, train_sl, train_y = input_data( train_set, max_sl) test_user, test_item, test_hist, test_sl, test_y = input_data( test_set, max_sl) current_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") log_dir = 'logs/' + current_time tensorboard = callbacks.TensorBoard(log_dir=log_dir, histogram_freq=1, write_graph=True, write_grads=False, write_images=True, embeddings_freq=0, embeddings_layer_names=None, embeddings_metadata=None, embeddings_data=None, update_freq=500) # model checkpoint check_path = 'save/din_weights.epoch_{epoch:04d}.val_loss_{val_loss:.4f}.ckpt' checkpoint = callbacks.ModelCheckpoint(check_path, save_weights_only=True, verbose=1, period=1) model = DIN(user_count, item_count, cate_count, cate_list, hidden_unit) model.summary() optimizer = optimizers.SGD(learning_rate=learning_rate, decay=0.1) model.compile(loss=losses.binary_crossentropy, optimizer=optimizer, metrics=[metrics.AUC()]) model.fit( [train_user, train_item, train_hist, train_sl], train_y, epochs=epochs, batch_size=batch_size, validation_split=0.1, # callbacks=[tensorboard, checkpoint] ) print('test AUC: %f' % model.evaluate([test_user, test_item, test_hist, test_sl], test_y))
def build_simple_model(dataset='Fashion Mnist', opt='sgd', hidden=None, funcs=None, loss=None, metrics_list=None): model = models.Sequential() if dataset == 'CIFAR-10': model.add(layers.Flatten(input_shape=[32, 32, 3])) elif ('Fashion Mnist'): model.add(layers.Flatten(input_shape=[28, 28])) for i in hidden.keys(): model.add(layers.Dense(hidden[i], activation=funcs[i].lower())) model.add(layers.Dense(10, activation="softmax")) loss_dict = { 'Categorical Crossentropy': 'categorical_crossentropy', 'Binary Crossentropy': 'binary_crossentropy', 'Categorical Hinge': 'categorical_hinge', 'Huber loss': 'huber_loss' } metrics_dict = { 'auc': metrics.AUC(), 'recall': metrics.Recall(), 'accuracy': metrics.CategoricalAccuracy() if loss.startswith('Categorical') else metrics.Accuracy(), 'precision': metrics.Precision(), 'categorical Hinge': metrics.CategoricalHinge(), 'squared Hinge': metrics.SquaredHinge(), 'Kullback-Leibler divergence': metrics.KLDivergence(), 'mean absolute error': metrics.MeanAbsoluteError(), 'mean squared error': metrics.MeanSquaredError() } if metrics_list is not None and len(metrics_list) > 0: metrics_list = [metrics_dict.get(m, m) for m in metrics_list] else: metrics_list = ['accuracy'] loss_f = loss_dict.get(loss) model.compile(loss=loss_f, optimizer=opt, metrics=metrics_list) return model
def inference(tfrecords_path, weights_path, wts_root): """ Inference function to reproduce original model scores. This script can be run as a standalone using python inference.py. For more information try: `python inference.py -h` Parameters ---------- tfrecords_path: str The path to directory containing preprocessed tfrecords. weights_path: str The path to the combined model weights. A copy of the weights can be found here: https://gin.g-node.org/shashankbansal56/nondefaced-detector-reproducibility/src/master/pretrained_weights/combined wts_root: str The path to the root directory of all the model weights. A copy of the weights can be found here: https://gin.g-node.org/shashankbansal56/nondefaced-detector-reproducibility/src/master/pretrained_weights """ model = CombinedClassifier(input_shape=(128, 128), dropout=0.4, wts_root=wts_root, trainable=False) model.load_weights(os.path.abspath(weights_path)) model.trainable = False dataset_test = get_dataset( file_pattern=os.path.join(tfrecords_path, "data-test_*"), n_classes=2, batch_size=16, volume_shape=(128, 128, 128), plane="combined", mode="test", ) METRICS = [ metrics.BinaryAccuracy(name="accuracy"), metrics.Precision(name="precision"), metrics.Recall(name="recall"), metrics.AUC(name="auc"), ] model.compile( loss=tf.keras.losses.binary_crossentropy, optimizer=Adam(learning_rate=1e-3), metrics=METRICS, ) model.evaluate(dataset_test)
def evaluate_model(model, dataset, precision_at_recall, recall_at_precision): """Evaluate the model and dataset. Args: model: A keras model with input (video_matrix, class_feature_list) and 1 output denoting class relevance dataset: tf.dataset attained from a Dataset class from readers.py precision_at_recall: the recall float value for which to calculate precision recall_at_precision: the precision float value for which to calculate recall """ segment_num = 0 #Create evaluation metrics aucroc_calculator = metrics.AUC() aucpr_calculator = metrics.AUC(curve='PR') pr_calculator = metrics.PrecisionAtRecall(precision_at_recall) rp_calculator = metrics.RecallAtPrecision(recall_at_precision) for input_data, label in dataset: prediction = evaluate_example(model, input_data) #Update Metrics aucroc_calculator.update_state(label, prediction) aucpr_calculator.update_state(label, prediction) pr_calculator.update_state(label, prediction) rp_calculator.update_state(label, prediction) print(f"Processing segment number {segment_num}") segment_num += 1 #Get results auc_roc = aucroc_calculator.result() auc_pr = aucpr_calculator.result() precision = pr_calculator.result() recall = rp_calculator.result() eval_dict = { "AUCPR": auc_pr, "AUCROC": auc_roc, "precision": precision, "recall": recall } return eval_dict
def __get_metric(self, metric): if metric == "auc": return m.AUC() elif metric == "accuracy": return m.Accuracy() elif metric == "binary_accuracy": return m.BinaryAccuracy() elif metric == "categorical_accuracy": return m.CategoricalAccuracy() elif metric == "binary_crossentropy": return m.BinaryCrossentropy() elif metric == "categorical_crossentropy": return m.CategoricalCrossentropy() elif metric == "sparse_categorical_crossentropy": return m.SparseCategoricalCrossentropy() elif metric == "kl_divergence": return m.KLDivergence() elif metric == "poisson": return m.Poission() elif metric == "mse": return m.MeanSquaredError() elif metric == "rmse": return m.RootMeanSquaredError() elif metric == "mae": return m.MeanAbsoluteError() elif metric == "mean_absolute_percentage_error": return m.MeanAbsolutePercentageError() elif metric == "mean_squared_logarithm_error": return m.MeanSquaredLogarithmError() elif metric == "cosine_similarity": return m.CosineSimilarity() elif metric == "log_cosh_error": return m.LogCoshError() elif metric == "precision": return m.Precision() elif metric == "recall": return m.Recall() elif metric == "true_positive": return m.TruePositives() elif metric == "true_negative": return m.TrueNegatives() elif metric == "false_positive": return m.FalsePositives() elif metric == "false_negative": return m.FalseNegatives() else: raise Exception("specified metric not defined")
def __init__(self, n_features, n_classes): print("##################### Init NN #####################") self.N_FEATURES = n_features self.N_CLASSES = n_classes self.METRICS = [ 'accuracy', tkm.TruePositives(), tkm.FalsePositives(name='fp'), tkm.TrueNegatives(name='tn'), tkm.FalseNegatives(name='fn'), #tkm.BinaryAccuracy(name='accuracy'), tkm.Precision(name='precision'), tkm.Recall(name='recall'), tkm.AUC(name='auc') ] self.DATE = datetime.now().strftime("%d-%m_%H%M%S") create_dir(self.DATE)
def main(input_dir, output_dir): class_names = [path.basename(s) for s in glob(input_dir + "/*/")] n_classes = len(class_names) # image_gen = preprocessing.image.ImageDataGenerator( # rescale=1.0 / 255, shear_range=0.2, zoom_range=0.2, horizontal_flip=True # ) image_gen = preprocessing.image.ImageDataGenerator(rescale=1.0 / 255) train_gen = image_gen.flow_from_directory( input_dir, target_size=(224, 224), batch_size=16, class_mode="categorical", # "binary" color_mode="rgb", ) base_model = applications.mobilenet_v2.MobileNetV2(input_shape=(224, 224, 3), include_top=False, weights="imagenet", pooling="max") for layer in base_model.layers: layer.trainable = False y = base_model.output y = layers.Dense(n_classes, activation="softmax")(y) model = models.Model(inputs=base_model.inputs, outputs=y) lr = 0.05 model.compile( optimizer=optimizers.Adam(lr), loss="categorical_crossentropy", metrics=["accuracy", metrics.AUC()], ) # print(model.summary()) model.fit( train_gen, validation_data=train_gen, epochs=20, callbacks=[ callbacks.ModelCheckpoint(output_dir, save_best_only=True), callbacks.EarlyStopping(patience=2), callbacks.LearningRateScheduler( lambda epoch: lr * np.exp(-0.1 * (epoch - 1))), callbacks.CSVLogger(path.join(output_dir, "metrics.csv")), ], )
def load_simple_model(model_path='', weights_path='', opt='sgd', loss=None, metrics_list=None): model = models.load_model(model_path) model.load_weights(weights_path) loss_dict = { 'Categorical Crossentropy': 'categorical_crossentropy', 'Binary Crossentropy': 'binary_crossentropy', 'Categorical Hinge': 'categorical_hinge', 'Huber loss': 'huber_loss' } metrics_dict = { 'auc': metrics.AUC(), 'recall': metrics.Recall(), 'accuracy': metrics.CategoricalAccuracy() if loss.startswith('Categorical') else metrics.Accuracy(), 'precision': metrics.Precision(), 'categorical Hinge': metrics.CategoricalHinge(), 'squared Hinge': metrics.SquaredHinge(), 'Kullback-Leibler divergence': metrics.KLDivergence(), 'mean absolute error': metrics.MeanAbsoluteError(), 'mean squared error': metrics.MeanSquaredError() } if metrics_list is not None and len(metrics_list) > 0: metrics_list = [metrics_dict.get(m, m) for m in metrics_list] else: metrics_list = ['accuracy'] loss_f = loss_dict.get(loss) model.compile(loss=loss_f, optimizer=opt, metrics=metrics_list) return model
def get_model(): # define convolutional layer # kernel size filters = 4 kernel_size = 4 convolution_1d_layer = keras.layers.Conv1D(filters, kernel_size, input_shape=(500, 1), strides=1, padding='same', activation="relu", name="convolution_1d_layer") # max pooling layer max_pooling_layer = keras.layers.MaxPooling1D(pool_size=2, strides=2, padding="same", name="max_pooling_layer") # reshape layer reshape_layer = keras.layers.Flatten(name="reshape_layer") # dropout layer dropout_layer = keras.layers.Dropout(0.5, name="dropout_layer") # full connect layer full_connect_layer = keras.layers.Dense(128, activation="relu", name="full_connect_layer") model = keras.Sequential() model.add(convolution_1d_layer) model.add(max_pooling_layer) model.add(reshape_layer) model.add(dropout_layer) model.add(full_connect_layer) model.add(layers.Dense(1, activation='sigmoid')) model.compile( optimizer=optimizers.Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-08), loss='binary_crossentropy', metrics=['accuracy',metrics.AUC()] ) return model
def experiment(self, under=False, ratio=3, plot=False): METRICS = [ metrics.TruePositives(name='tp'), metrics.FalsePositives(name='fp'), metrics.TrueNegatives(name='tn'), metrics.FalseNegatives(name='fn'), metrics.BinaryAccuracy(name='accuracy'), metrics.Precision(name='precision'), metrics.Recall(name='recall'), metrics.AUC(name='auc') ] data = DataLoader() model = LeNet(data.X, METRICS) augmenter = Augmenter(data.X, data.Y) if under: data.X, data.Y = augmenter.undersample(ratio=ratio) if self.augmentation.type == 1 or self.augmentation.type == 2: data.X, data.Y = augmenter.duplicate(noise=self.augmentation.noise, sigma=self.augmentation.sigma) elif self.augmentation.type == 3: data.X, data.Y = augmenter.SMOTE() #data.normalize() #print(len(data.X)) #print(len(data.valX)) data.summarize(test=False) his = model.fit(data.X, data.Y, data.valX, data.valY) RES, fpr, tpr = model.predict(data.testX, data.testY) #self.model_summary(RES) if plot: self.plot(his) self.ROC(fpr, tpr) return RES
def train(args): input_shape = (args.img_size, args.img_size, 3) train_gen, valid_gen = get_generators( args.data_path, args.img_size, args.batch_size, args.test_img ) model = get_model(input_shape, args.n_classes) model.compile( RMSprop(lr=args.learning_rate), loss="categorical_crossentropy", metrics=[metrics.AUC(name='auc'), 'accuracy'] ) es_callback = EarlyStopping( monitor='val_auc', mode='max', patience=5, verbose=1, min_delta=0.005, restore_best_weights=True ) history = model.fit( train_gen, steps_per_epoch=train_gen.samples//args.batch_size, epochs = args.num_epochs, validation_data=valid_gen, validation_steps=valid_gen.samples//args.batch_size, callbacks= [es_callback], verbose=1 ) model.save_weights(f"{args.weigths_path}/fish_classification")
def compile(self, model, train_generator, valid_generator): """:arg This function contain model compile and model fit process, input a model and output history and trained model """ start_time = time() print("*" * 40, "Start {} Processing".format(model._name), "*" * 40) # we use a lot of metric to evalute our binary classification result METRICS = [ metrics.TruePositives(name='tp'), metrics.FalsePositives(name='fp'), metrics.TrueNegatives(name='tn'), metrics.FalseNegatives(name='fn'), metrics.BinaryAccuracy(name='binary_accuracy'), #metrics.CategoricalAccuracy(name='accuracy'), metrics.Precision(name='precision'), metrics.Recall(name='recall'), metrics.AUC(name='auc'), # F1Score(num_classes = int(y_train.shape[1]), name='F1') ] # define a optimizer opt_rms = optimizers.RMSprop(lr = 1e-4, decay = 1e-5) # define compile parameters model.compile(loss = 'binary_crossentropy', optimizer = opt_rms, metrics = ['accuracy']) # start to fit history = model.fit( train_generator, steps_per_epoch=20, epochs=5, validation_data=valid_generator, validation_steps=20 ) return history
def cross_validate(self, train_data_list, **build_kwargs): """ Performs cross-validation on the model. Parameters ---------- train_data_list: list A list where each element contains multiple images. (Each element corresponds to a fold) Returns ------- Model A trained model """ print(f"Cross validating {str(self)}") # Create lists for holding the metric values acc_per_fold = [] # accuracy auc_per_fold = [] # AUC loss_per_fold = [] # loss # Create 2 empty lists for holding the training history losses and test history losses fold_train_loss, fold_test_loss = list(), list() # Iterate an index to size of folds for i in range(len(train_data_list)): # Extract validation data x_val, y_val = train_data_list[i] # Indices for train indices_to_keep = np.delete(range(len(train_data_list)), i) # Get train data x_train, y_train = _build_train_data(train_data_list, indices_to_keep) # Compute class weights for balanced learning. Returns a list weights = compute_class_weight("balanced", classes=np.unique(y_train), y=y_train) # Convert the list do dict. weights_dict = {idx: value for idx, value in enumerate(weights)} # Build the model baseline_model = self.build( input_shape=x_train[0].shape, **build_kwargs, ) # Compile the model baseline_model.compile( optimizer=optimizers.Adam(), loss=losses.BinaryCrossentropy(), metrics=[metrics.BinaryAccuracy(), metrics.AUC(name="auc")], ) # Train the model with the training indices history = baseline_model.fit( x_train, y_train, validation_data=(x_val, y_val), batch_size=64, epochs=40, class_weight=weights_dict, verbose=0, ) fold_train_loss.append(history.history["loss"]) fold_test_loss.append(history.history["val_loss"]) # Evaluate the model with the testing indices scores = baseline_model.evaluate(x_val, y_val, verbose=0) # Scores has format : ['loss', 'binary_accuracy', 'auc'] # Save the loss value on the val data loss_per_fold.append(scores[0]) # Save the Accuracy value on the val data acc_per_fold.append(scores[1]) # Save the Accuracy value on the val data auc_per_fold.append(scores[2]) # Plot the loss curve self._plot_losses( fold_train_loss, fold_test_loss, ) # Display the results print( f"Accuracy: {acc_per_fold.mean():.2f} (+/- {acc_per_fold.std():.2f})" ) print(f"AUC: {auc_per_fold.mean():.2f} (+/- {auc_per_fold.std():.2f})") return baseline_model
def define_model(self, length, vocab_size, num_outcome_classes): ''' Defines and compiles a convolutional neural network model Parameters ___________ length: int Max length of the text strings vocab_size: int Vocabulary size of the text documents ''' model = Sequential() model.add(Input(shape=(length,))) model.add(Embedding(vocab_size, 250)) model.add(Dropout(.25)) model.add(Bidirectional(GRU(64))) model.add(Dropout(.5)) model.add(Dense(32, activation='relu')) model.add(Dense(num_outcome_classes, activation='softmax')) model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['categorical_accuracy', metrics.AUC()]) self.model = model
def train_model(self, themes_weight: List[float], dataset: TrainValidationDataset, voc_size: int, keras_callback: LambdaCallback): article_length = dataset.article_length theme_count = dataset.theme_count model = tf.keras.Sequential([ # 1 # keras.layers.Embedding(input_dim=voc_size, output_dim=firstLayoutOutputDim), # keras.layers.Dropout(0.2), # keras.layers.Conv1D(200,3,input_shape=(ARTICLE_MAX_WORD_COUNT,firstLayoutOutputDim), activation=tf.nn.relu), # keras.layers.GlobalAveragePooling1D(), # keras.layers.Dense(250, activation=tf.nn.relu), # keras.layers.Dense(theme_count, activation=tf.nn.softmax) # 2 # keras.layers.Embedding(input_dim=voc_size, output_dim=firstLayoutOutputDim), # keras.layers.LSTM(ltsmOutputDim, dropout=0.2, recurrent_dropout=0.2, activation='tanh'), # keras.layers.Dense(theme_count, activation=tf.nn.softmax) # 3 # keras.layers.Embedding(input_dim=self.voc_size, output_dim=embedding_output_dim), # keras.layers.Bidirectional(keras.layers.LSTM(intermediate_dim, return_sequences=True)), # # keras.layers.Dropout(0.1), # keras.layers.Bidirectional(keras.layers.LSTM(last_dim, dropout=0.05, recurrent_dropout=0.05)), # keras.layers.Dense(last_dim, activation=tf.nn.relu), # keras.layers.Dense(self.theme_count, activation=tf.nn.softmax) # 4 # keras.layers.Embedding(input_dim=self.voc_size, input_length=self.article_length, output_dim=embedding_output_dim), # keras.layers.Bidirectional(keras.layers.LSTM(intermediate_dim, return_sequences=True, dropout=0.2, recurrent_dropout=0.2)), # keras.layers.Dropout(0.2), # keras.layers.Bidirectional(keras.layers.LSTM(last_dim * 2, recurrent_dropout=0.2)), #was last_dim * 2 # keras.layers.Dense(last_dim, activation=tf.nn.relu), # keras.layers.Dense(self.theme_count, activation=tf.nn.sigmoid) # 5 #keras.layers.Embedding(input_dim=self.voc_size, input_length=self.article_length, output_dim=embedding_output_dim), # keras.layers.Conv1D(filters=64, kernel_size=5, input_shape=(self.voc_size, embedding_output_dim), activation="relu"), # keras.layers.MaxPool1D(4), #keras.layers.Bidirectional(keras.layers.LSTM(intermediate_dim, recurrent_dropout=0.1)), #keras.layers.Dense(last_dim, activation=tf.nn.relu), #keras.layers.Dense(self.theme_count, activation=tf.nn.sigmoid) #6 keras.layers.Embedding(input_dim=voc_size, input_length=article_length, output_dim=128, mask_zero=True), keras.layers.Bidirectional( keras.layers.LSTM(128, recurrent_dropout=0.2, dropout=0.2)), #keras.layers.Dropout(0.2), #keras.layers.Dense(last_dim, activation=tf.nn.relu), # keras.layers.Dense(self.theme_count, activation=tf.nn.sigmoid, use_bias=True,bias_initializer=tf.keras.initializers.Constant(-1.22818328)) keras.layers.Dense(theme_count, activation=tf.nn.sigmoid, kernel_regularizer=regularizers.l2(0.1), activity_regularizer=regularizers.l1(0.05)) # 7 # keras.layers.Embedding(input_dim=self.voc_size, input_length=self.article_length, # output_dim=embedding_output_dim), # keras.layers.GlobalAvgPool1D(), # keras.layers.Dense(last_dim, activation=tf.nn.relu), # keras.layers.Dense(self.theme_count, activation=tf.nn.sigmoid) ]) model.summary() model.compile( optimizer=tf.keras.optimizers.Adam(clipnorm=1, clipvalue=0.5), #loss=WeightedBinaryCrossEntropy(themes_weight, from_logits=True), loss=keras.losses.BinaryCrossentropy(from_logits=True), metrics=[ metrics.AUC(), metrics.BinaryAccuracy(), metrics.TruePositives(), metrics.TrueNegatives(), metrics.FalseNegatives(), metrics.FalsePositives(), metrics.Recall(), metrics.Precision() ], run_eagerly=self.run_eagerly) keras.utils.plot_model(model, 'Model1.png', show_shapes=True) cb_list = [ManualInterrupter, keras_callback] model.fit(dataset.trainData, epochs=10, steps_per_epoch=dataset.train_batch_count, validation_data=dataset.validationData, validation_steps=dataset.validation_batch_count, callbacks=cb_list, class_weight={ 0: 1, 1: themes_weight[0] }) model.save("output/" + self.get_model_name() + ".h5") model.save_weights("output/" + self.get_model_name() + "_weight.h5") self.__model__ = model
def call(self, local_true, local_pred): local_true = tf.convert_to_tensor(local_true, dtype=tf.float32) local_pred = tf.convert_to_tensor(local_pred, dtype=tf.float32) local_auc_roc = tf.math.add(tf.math.add(1., metrics.AUC(local_true, local_pred))) return local_auc_roc
} if __name__ == "__main__": if len(argv) != 2 or argv[1] not in networks.keys(): options = "|".join(networks.keys()) print(f"usage: ./predict_review.py {options}") else: name = argv[1] (X_train, Y_train, X_dev, Y_dev, X_test, Y_test) = prepare( filename="./data/02_filter.json", max_ingredients=int(networks[name]["max_ingredients"]), ) model = Sequential(networks[name]["layers"]) model.compile( loss=losses.binary_crossentropy, metrics=[metrics.AUC()], optimizer=optimizers.Adam(), ) run_time = datetime.now().strftime("%Y%m%d-%H%M%S") training = model.fit( x=X_train, y=Y_train, batch_size=BATCH_SIZE, callbacks=[ callbacks.TensorBoard( log_dir=LOG_DIR.format(network=name, run=run_time), histogram_freq=1, ) ], epochs=5, validation_data=(X_dev, Y_dev),
IMGSIZE = (int(args.imgsize[0]), int(args.imgsize[1])) LOGDIR = args.logdir DATA = args.data BACKBONE = args.backbone NAME = args.model # --- define model metrics --- METRICS = [ metrics.TruePositives(name="True_Positives"), metrics.FalsePositives(name="False_Positives"), metrics.TrueNegatives(name="True_Negatives"), metrics.FalseNegatives(name="False_Negatives"), metrics.BinaryAccuracy(name="Binary_Accuracy"), metrics.Precision(name="Precision"), metrics.Recall(name="Recall"), metrics.AUC(name="AUC") ] # --- tensorflow calbacks --- date = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") if platform.system().lower() == "windows": LOGDIR = LOGDIR + "\\" + NAME + "\\" + date else: LOGDIR = LOGDIR + "/" + NAME + "/" + date if not os.path.isdir(LOGDIR): os.makedirs(LOGDIR, exist_ok=True) tensorboard = callbacks.TensorBoard(log_dir=LOGDIR, histogram_freq=1, profile_batch=1)
# Encode labels onehot = mlu.EncoderHelper() onehot.fit(y_train) y_train = onehot.transform(y_train) y_test = onehot.transform(y_test) # Create model model = create_model() model.compile( optimizer="adam", loss="categorical_crossentropy", metrics=[ met.CategoricalAccuracy(name="categorical_accuracy"), met.Recall(name="recall"), met.Precision(name="precision"), met.AUC(name="auc"), tfa_met.F1Score(num_classes=num_classes, average="macro", name="f1_macro"), tfa_met.MatthewsCorrelationCoefficient(num_classes=num_classes, name="mcc"), ], ) # Fit data # tfu.kill_tensorboard_pids() proc = tfu.start_tensorboard(tfu.get_tb_logdir("gpu_test", False)) model.fit( x=x_train, y=y_train, epochs=5,
X_train, X_test, y_train, y_test = train_test_split(trainLines, transformed_labels, test_size=.2, random_state=42, stratify=transformed_labels) length = encoder.max_length(X_train) vocab_size = encoder.vocab_size(X_train) X_train = encoder.encode_text(X_train) X_test = encoder.encode_text(X_test, test_data=True) inputs1 = Input(shape=(length,)) embedding1 = Embedding(vocab_size, 100)(inputs1) conv1 = Conv1D(filters=32, kernel_size=4, activation='relu')(embedding1) drop1 = Dropout(0.5)(conv1) pool1 = MaxPooling1D(pool_size=2)(drop1) flat1 = Flatten()(pool1) dense1 = Dense(64, activation='relu')(flat1) outputs = Dense(3, activation='softmax')(dense1) model = Model(inputs=inputs1, outputs=outputs) model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy', metrics.AUC()]) model.fit(X_train, y_train, validation_data=(X_test, y_test), epochs=10, batch_size=128) test_text = "I have been an every dollar user for the past 18 months and occasionally have issues loading my USAA accounts. I followed your troubleshooting page and have deleted my bank account and attempted to reload it but hasn't loaded. Any help would be appreciated." test_text = ct.prepare_text(test_text) test_text = encoder.encode_text(test_text, test_data=True) model.predict(test_text)
dropout=0.3) # Build the model and expose input and output sockets of graphsage model # for link prediction x_inp, x_out = graphsage.in_out_tensors() prediction = link_classification(output_dim=1, output_act="relu", edge_embedding_method="ip")(x_out) model = keras.Model(inputs=x_inp, outputs=prediction) model.compile( optimizer=keras.optimizers.Adam(lr=1e-3), loss=keras.losses.binary_crossentropy, metrics=[tfm.AUC(), "acc", utils.get_f1, utils.get_pres, utils.get_rec]) init_train_metrics = model.evaluate(train_flow) init_test_metrics = model.evaluate(test_flow) print("\nTrain Set Metrics of the initial (untrained) model:") for name, val in zip(model.metrics_names, init_train_metrics): print("\t{}: {:0.4f}".format(name, val)) print("\nTest Set Metrics of the initial (untrained) model:") for name, val in zip(model.metrics_names, init_test_metrics): print("\t{}: {:0.4f}".format(name, val)) epochs = 20 history = model.fit(train_flow,