def test_unweighted_with_threshold(self): p_obj = metrics.Precision(thresholds=[0.5, 0.7]) y_pred = K.constant([1, 0, 0.6, 0], shape=(1, 4)) y_true = K.constant([0, 1, 1, 0], shape=(1, 4)) result = p_obj(y_true, y_pred) assert np.allclose([0.5, 0.], K.eval(result), 0)
def run(model_name, train_file, val_file, num_classes, filename, dropout, input_shape_arg = (224,224,3)): """ fit dataset and run training process Arguments:\n train_file --> h5 file, fit to model\n val_file --> h5 file, for validation\n num_classes --> int, total classes \n dropout_value --> float, range 0 - 1 for dropout\n epoch --> int\n batch_size --> int, [8, 16, 32, 64, 128, 256, etc.]\n input_shape_arg --> shape of image (W,H,C)\n lr_value --> learning rate value\n optimizer --> Adam, SGD\n Returns:\n model\n x_test\n y_test """ # preprocessing data X_train, Y_train, X_val, Y_val = dataset_preprocess(num_classes, train_file, val_file) _epoch = 80 lr_value_array = [1e-3, 1e-4] if model_name == "resnet50": batch_size_array = [8, 16, 32] LABEL = ["e-3(8)", "e-3(16)", "e-3(32)", "e-4(8)", "e-4(16)", "e-4(32)"] elif model_name == "resnet18": batch_size_array = [16, 32, 64] LABEL = ["e-3(16)", "e-3(32)", "e-3(64)", "e-4(16)", "e-4(32)", "e-4(64)"] HP = [] for lr in lr_value_array: for bs in batch_size_array: hp = Hyperparameter("adam", lr, bs, dropout) HP.append(hp) HISTORY = [] ROC = [] for hp in HP: K.clear_session() model = None # compile model if model_name == "resnet50": print("resnet50") model = ResNet50(input_shape=input_shape_arg, classes=int(num_classes), dropout_value=hp.dropout) model.compile(optimizer=hp.get_optimizer(), loss='categorical_crossentropy', metrics=['accuracy', metrics.AUC(), metrics.Precision(), metrics.Recall()]) elif model_name == "resnet18": print("resnet18") model = ResNet18(input_shape=input_shape_arg, classes=int(num_classes), dropout_value=hp.dropout) model.compile(optimizer=hp.get_optimizer(), loss='categorical_crossentropy', metrics=['accuracy', metrics.AUC(), metrics.Precision(), metrics.Recall()]) elif model_name == "vgg19": print("VGG19") # configure model input base_model = applications.vgg19.VGG19(weights= None, include_top=False, input_shape= input_shape_arg) # configure model output x = base_model.output x = GlobalAveragePooling2D()(x) x = Dropout(hp.dropout(x)) out = Dense(int(num_classes), activation= 'softmax')(x) # combine model then compile model = Model(inputs = base_model.input, outputs = out) model.compile(optimizer= hp.get_optimizer(), loss='categorical_crossentropy', metrics=['accuracy']) elif model_name == "vgg16": print("VGG16") # configure model input base_model = applications.vgg16.VGG16(weights= None, include_top=False, input_shape= input_shape_arg) # configure model output x = base_model.output x = GlobalAveragePooling2D()(x) x = Dropout(hp.dropout(x)) out = Dense(int(num_classes), activation= 'softmax')(x) # combine model then compile model = Model(inputs = base_model.input, outputs = out) model.compile(optimizer= hp.get_optimizer(), loss='categorical_crossentropy', metrics=['accuracy']) # optimizer == adam # train the model history = model.fit(X_train, Y_train, epochs = _epoch, batch_size = hp.batch_size, validation_data=(X_val, Y_val), shuffle=True) HISTORY.append(history) del model print(f"DONE for: {hp.optim}-{hp.lr_value}-{hp.batch_size}-{hp.dropout}") plt.figure(1) mpl.style.use('seaborn') i = 0 for history in HISTORY: plt.plot(history.history["val_accuracy"], f"C{i}", label=LABEL[i]) i = i+1 plt.ylabel('val_acc') plt.xlabel('epoch') plt.title(f"Accuracy {filename}") plt.legend() plt.savefig(f"ACC-{filename}.png") print("VAL ACC:") i = 0 for history in HISTORY: print(LABEL[i]) i = i + 1 print("auc: {}" .format(statistics.mean( history.history["val_auc_1"] )) ) print("recall: {}" .format(statistics.mean( history.history["val_recall_1"] )) ) print("Prec: {}" .format(statistics.mean( history.history["val_precision_1"] )) ) print("MEAN: {}" .format(statistics.mean( history.history["val_accuracy"] )) ) print("STD: {}" .format(statistics.pstdev( history.history['val_accuracy'] )) )
def test_unweighted(self): p_obj = metrics.Precision() y_pred = K.constant([1, 0, 1, 0], shape=(1, 4)) y_true = K.constant([0, 1, 1, 0], shape=(1, 4)) result = p_obj(y_true, y_pred) assert np.isclose(0.5, K.eval(result))
def set_model_prediction_multi_label(model_path, test_set, custom_objects, with_f1=False): """Compute metrics of a set of models for multilabel model. Computed metrics are : Loss, Accuracy, F1-Score, Macro F1-Score. Args: model_path (string): path of folder that contains models. test_set (pandas.DataFrame): DataFrame with column path for images path and label. custom_objects (dict): Dict of custom objects to load with model. with_f1 (bool, optional): True if the model is train with F1-Score metrics, otherwise False. Defaults to False. """ test_generator = generator(test_set['path'].to_numpy(), test_set[['label_culture','label_coffee']].to_numpy(), eurosat_params['mean'], eurosat_params['std'], batch_size=len(test_set)) prediction_set = [] evaluate = [] X, y = next(test_generator) for path in os.listdir(model_path): if path.split(".")[1] == 'h5': restored_model = None if with_f1: restored_model = load_model(os.path.join(model_path, path), custom_objects, compile=False) restored_model.compile(optimizer=Adam(learning_rate=0.00001), loss='binary_crossentropy',metrics=[metrics.BinaryAccuracy(name='accuracy'),metrics.Precision(name='precision'),metrics.Recall(name='recall'),f1_score_keras]) else : restored_model = load_model(os.path.join(model_path, path), custom_objects) evaluate.append(restored_model.evaluate(test_generator, steps=1)) prediction_set.append(np.where(restored_model.predict(X) > 0.5, 1, 0)) predictions = [] for pred in zip(*prediction_set): culture_pred, coffee_pred = zip(*pred) predictions.append(np.array([np.argmax(np.bincount(culture_pred)), np.argmax(np.bincount(coffee_pred))])) cm = multilabel_confusion_matrix(y, predictions) plot_confusion_matrix(cm[0], ["Culture", "No-Culture"],"Confusion Matrix\nCulture vs No-Culture\nDenseNet 64x64") plot_confusion_matrix(cm[1], ["Coffee", "Other"],"Confusion Matrix\nCoffee vs other\nDenseNet 64x64") if with_f1: losses, accs, precisions, recalls, f1 = zip(*evaluate) else : losses, accs, precisions, recalls = zip(*evaluate) print("Global metrics") print(f"Mean accuracy : {round(np.mean(accs),4)}") print(f"Stdev accuracy : {round(np.std(accs),4)}") print("\n") print(f"Mean loss : {round(np.mean(losses),4)}") print(f"Stdev loss : {round(np.std(losses),4)}") print("\n") culture_pred, coffee_pred = zip(*predictions) culture_true, coffee_true = zip(*y) print("Culture vs no-culture") print(f"F1-Score Culture: {round(f1_score(culture_true, culture_pred, pos_label=0),4)}") print(f"F1-Score No culture: {round(f1_score(culture_true, culture_pred, pos_label=1),4)}") print(f"Macro F1-Score : {round(f1_score(culture_true, culture_pred, average='macro'),4)}") print(f"\n") print("Coffee vs other") print(f"F1-Score Coffee: {round(f1_score(coffee_true, coffee_pred, pos_label=0),4)}") print(f"F1-Score Other: {round(f1_score(coffee_true, coffee_pred, pos_label=1),4)}") print(f"Macro F1-Score : {round(f1_score(coffee_true, coffee_pred, average='macro'),4)}")
model.add(Dropout(dropout_9)) model.add(Dense(units_10, activation=activation_10)) model.add(Dropout(dropout_10)) model.add(Dense(units_11, activation=activation_11)) model.add(Dropout(dropout_11)) model.add(Dense(units_12, activation=activation_12)) model.add(Dropout(dropout_12)) model.add(Dense(nb_classes, activation='sigmoid')) METRICS = [ metrics.BinaryAccuracy(name='ACCURACY'), metrics.Precision(name='PRECISION'), metrics.Recall(name='RECALL'), metrics.AUC(name='AUC'), metrics.TruePositives(name='TP'), metrics.TrueNegatives(name='TN'), metrics.FalsePositives(name='FP'), metrics.FalseNegatives(name='FN')] model.compile(loss='binary_crossentropy', optimizer=compile_optimizer, metrics=METRICS) # GENERATORS train_datagen = ImageDataGenerator( rescale=1. / 255, shear_range=0.2,
def set_model_prediction(model_path, test_set, custom_objects, with_f1=False, labels=["Coffee", "Other"], title="title"): """Compute metrics of a set of models. Computed metrics are : Loss, Accuracy, F1-Score, Macro F1-Score. Args: model_path (string): path of folder that contains models. test_set (pandas.DataFrame): DataFrame with column path for images path and label. custom_objects (dict): Dict of custom objects to load with model. with_f1 (bool, optional): True if the model is train with F1-Score metrics, otherwise False. Defaults to False. labels (list, optional): Label for confusion Matrix. Defaults to ["Coffee", "Other"]. title (str, optional): Title of confusion matrix. Defaults to "title". """ test_generator = generator(test_set['path'].to_numpy(), test_set['label'].to_numpy(), eurosat_params['mean'], eurosat_params['std'], batch_size=len(test_set)) prediction_set = [] evaluate = [] X, y = next(test_generator) for path in os.listdir(model_path): if path.split(".")[1] == 'h5': restored_model = None if with_f1: restored_model = load_model(os.path.join(model_path, path), custom_objects, compile=False) restored_model.compile(optimizer=Adam(learning_rate=0.00001), loss='binary_crossentropy',metrics=[metrics.BinaryAccuracy(name='accuracy'),metrics.Precision(name='precision'),metrics.Recall(name='recall'),f1_score_keras]) else : restored_model = load_model(os.path.join(model_path, path), custom_objects) evaluate.append(restored_model.evaluate(test_generator, steps=1)) prediction_set.append(np.where(restored_model.predict(X) > 0.5, 1, 0).reshape(-1).tolist()) predictions = [] for pred in zip(*prediction_set): predictions.append(np.argmax(np.bincount(pred))) cm = confusion_matrix(y, predictions) plot_confusion_matrix(cm, labels,title) if with_f1: losses, accs, precisions, recalls, f1 = zip(*evaluate) else : losses, accs, precisions, recalls = zip(*evaluate) print(f"Mean accuracy : {round(np.mean(accs),4)}") print(f"Stdev accuracy : {round(np.std(accs),4)}") print("\n") print(f"Mean loss : {round(np.mean(losses),4)}") print(f"Stdev loss : {round(np.std(losses),4)}") print("\n") print(f"F1-Score {labels[0]}: {round(f1_score(y, predictions, pos_label=0),4)}") print(f"F1-Score {labels[1]}: {round(f1_score(y, predictions, pos_label=1),4)}") print(f"Macro F1-Score : {round(f1_score(y, predictions, average='macro'),4)}")
print(imerg.shape, label.shape,goes.shape, flush=True) model=UNet() print(model.summary(),flush=True) model = multi_gpu_model(model,gpus=2) metrics = [ metrics.FalseNegatives(name="fn"), metrics.FalsePositives(name="fp"), metrics.TrueNegatives(name="tn"), metrics.TruePositives(name="tp"), metrics.Precision(name="precision"), metrics.Recall(name="recall"), ] model.compile(optimizer="Adam", loss='binary_crossentropy', metrics=metrics) epochs=500 batch_size=16 earlystopper = EarlyStopping(patience=50,verbose=1, monitor='val_loss') checkpointer = ModelCheckpoint('model_ck_mse_new.h5', save_best_only=True, verbose=1) history=model.fit([goes,imerg], label, epochs=epochs, batch_size=batch_size, validation_split=0.3, callbacks=[earlystopper,checkpointer], verbose=2) results=pd.DataFrame(history.history)
def f1_score(y_true, y_pred): return 2 * ( (metrics.Precision(y_true, y_pred) * metrics.Recall(y_true, y_pred)) / (metrics.Precision(y_true, y_pred) + metrics.Recall(y_true, y_pred))) + k.epsilo()
def train_model(dataset, model): epochs = 50 # epochs = 0 lr = 1e-3 size = 300 wd = 1e-2 bs = 4 # reduce this if you are running out of GPU memory pretrained = True config = { 'epochs': epochs, 'lr': lr, 'size': size, 'wd': wd, 'bs': bs, 'pretrained': pretrained, } wandb.config.update(config) checkpointer = ModelCheckpoint('model-resnet50.h5', verbose=1, save_best_only=True) # # Define IoU metric # def mean_iou(y_true, y_pred): # prec = [] # for t in np.arange(0.5, 1.0, 0.05): # y_pred_ = tf.to_int32(y_pred > t) # score, up_opt = tf.metrics.mean_iou(y_true, y_pred_, 2) # K.get_session().run(tf.local_variables_initializer()) # with tf.control_dependencies([up_opt]): # score = tf.identity(score) # prec.append(score) # return K.mean(K.stack(prec), axis=0) earlystopper = EarlyStopping(patience=5, verbose=1) learning_rate_reduction = ReduceLROnPlateau(monitor='val_loss', patience=3, verbose=1, factor=0.5, min_lr=0.00001) model.compile(optimizer=optimizers.Adam(lr=lr), loss='categorical_crossentropy', metrics=[ metrics.Precision(top_k=1, name='precision'), metrics.Recall(top_k=1, name='recall'), FBeta(name='f_beta') ]) train_data, valid_data = datasets_keras.load_dataset(dataset, bs) _, ex_data = datasets_keras.load_dataset(dataset, 10) model.fit_generator(train_data, validation_data=valid_data, epochs=epochs, callbacks=[ earlystopper, learning_rate_reduction, checkpointer, WandbCallback(input_type='image', output_type='segmentation_mask', validation_data=ex_data[0]) ])
def test_unweighted_top_k(self): p_obj = metrics.Precision(top_k=3) y_pred = K.constant([0.2, 0.1, 0.5, 0, 0.2], shape=(1, 5)) y_true = K.constant([0, 1, 1, 0, 0], shape=(1, 5)) result = p_obj(y_true, y_pred) assert np.isclose(1. / 3, K.eval(result))