def score_model_best_iou(net, X_valid, y_valid, device, debug=False):
"""
Scores the model and do a threshold optimization by the best IoU.
Arguments:
net: The network to be evaluated
X_valid: The data samples for validation
y_valid: The ground truth data for validation
device: The Torch device to use
debug: The flag to indicate if debug info should be displayed
Returns:
(threshold_best, iou_best) the threshold for best IoU metric and best IoU metric value
"""
net.eval()
with torch.no_grad():
hebb = net.initialZeroHebb()
# Find predictions for validation
preds_valid = []
for x in X_valid:
t_img = torch.from_numpy(np.array([x.astype(np.float32)
])).to(device)
# We do not learn plasticity within validation
mask_pred, _ = net(Variable(t_img, requires_grad=False),
Variable(hebb, requires_grad=False))
preds_valid.append(mask_pred.cpu().numpy())
# Scoring model, choose threshold by validation data
thresholds_ori = np.linspace(0.3, 0.7, 31)
# Reverse sigmoid function: Use code below because the sigmoid activation was removed
thresholds = np.log(thresholds_ori / (1 - thresholds_ori))
ious = np.array([
iou_metric_batch(y_valid, preds_valid > threshold)
for threshold in thresholds
])
if debug:
print(ious)
# instead of using default 0 as threshold, use validation data to find the best threshold.
threshold_best_index = np.argmax(ious)
iou_best = ious[threshold_best_index]
threshold_best = thresholds[threshold_best_index]
if debug:
plot_best_iou(thresholds=thresholds, ious=ious)
return threshold_best, iou_best
break
y_pred_test = np.array(y_pred_test)
if (args.debug):
print("type(y_pred_test) : ", type(y_pred_test))
print("y_pred_test.shape : ", y_pred_test.shape)
#================================
# 可視化処理
#================================
# 最適なマスクスレッショルド値での IoU スコアの計算
thresholds = np.linspace(
-0.95, 0.95,
100) # IoU スコアの低い結果を除外するためのスレッショルド（-1.0 ~ 1.0 は生成マスク画像のピクセル値に対応）
ious = np.array([
iou_metric_batch(y_pred_valid_mask, np.int32(y_pred_valid > threshold))
for threshold in thresholds
])
threshold_best_index = np.argmax(ious[9:-10]) + 9
iou_best = ious[threshold_best_index]
threshold_best = thresholds[threshold_best_index]
print("iou_best = {:0.4f} ".format(iou_best))
print("threshold_best = {:0.4f} ".format(threshold_best))
fig, axs = plt.subplots()
axs.plot(thresholds, ious)
axs.plot(threshold_best, iou_best, "xr", label="Best threshold")
plt.xlabel("Threshold (mask pixel value)")
plt.ylabel("IoU")
plt.title("Threshold vs IoU ({}, {})".format(threshold_best, iou_best))
########################################
#### ####
#### SCORING ####
#### ####
########################################
model = load_model(f'{output_folder_path}/models/{basic_name}-stage2.model',
custom_objects=custom_objects)
preds_valid = predict_result(model, x_valid)
## Scoring for last model, choose threshold by validation data
thresholds_ori = np.linspace(0.3, 0.7, 31)
# Reverse sigmoid function: Use code below because the sigmoid activation was removed
thresholds = np.log(thresholds_ori/(1-thresholds_ori))
ious = np.array([iou_metric_batch(y_valid, preds_valid > threshold) for threshold in tqdm(thresholds)])
# instead of using default 0 as threshold, use validation data to find the best threshold.
threshold_best_index = np.argmax(ious)
iou_best = ious[threshold_best_index]
threshold_best = thresholds[threshold_best_index]
log_metric('Best IoU', iou_best)
log_metric('Best Threshold', threshold_best)
plt.plot(thresholds, ious)
plt.plot(threshold_best, iou_best, "xr", label="Best threshold")
plt.xlabel("Threshold")
plt.ylabel("IoU")
plt.title(f"Threshold vs IoU ({threshold_best}, {iou_best})")
model = models.Model(input_layer, output_layer)
model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['acc'])
es = callbacks.EarlyStopping(patience=30, verbose=1, restore_best_weights=True)
rlp = callbacks.ReduceLROnPlateau(factor=0.1,
patience=5,
min_lr=1e-12,
verbose=1)
results = model.fit(X_train,
Y_train,
validation_data=(X_eval, Y_eval),
batch_size=8,
epochs=1,
callbacks=[es, rlp])
preds_eval = model.predict(X_eval, verbose=1)
thresholds = np.linspace(0, 1, 50)
ious = np.array([
iou_metric_batch(Y_eval, np.int32(preds_eval > threshold))
for threshold in tqdm(thresholds)
])
threshold_best_index = np.argmax(ious[9:-10]) + 9
iou_best = ious[threshold_best_index]
threshold_best = thresholds[threshold_best_index]
print(iou_best)
print(threshold_best)