def find_best_threshold(labels, probabilities):
threshold = np.zeros(17)
acc = 0
# iterate over transformations
for t_idx in range(len(transforms)):
# iterate over class labels
t = np.ones(17) * 0.15
selected_preds = probabilities[t_idx]
selected_preds = np.mean(selected_preds, axis=0)
best_thresh = 0.0
best_score = 0.0
for i in range(17):
for r in range(500):
r /= 500
t[i] = r
preds = (selected_preds > t).astype(int)
score = f2_score(labels, preds)
if score > best_score:
best_thresh = r
best_score = score
t[i] = best_thresh
threshold = threshold + t
acc += best_score
print('AVG ACC,', acc / len(transforms))
threshold = threshold / len(transforms)
return threshold
def optimize_threshold(models, datasets, resolution=1000):
"""
This function takes the validation set and find the best threshold for each class.
"""
pred = []
targets = []
# predict
# for index, data in enumerate(zip(*datasets)):
for batch_index, (image, target, _) in enumerate(datasets[0]):
image = Variable(image.cuda(), volatile=True)
output = F.sigmoid(models[0](image))
pred.append(output.data.cpu().numpy())
targets.append(target.cpu().numpy())
pred = np.vstack(pred)
targets = np.vstack(targets)
threshold = [0.15] * 17
# optimize
for i in range(17):
best_thresh = 0.0
best_score = 0.0
for r in range(resolution):
r /= resolution
threshold[i] = r
labels = (pred > threshold).astype(int)
score = f2_score(targets, labels)
if score > best_score:
best_thresh = r
best_score = score
threshold[i] = best_thresh
print(i, best_score, best_thresh)
return threshold
def find_best_threshold(labels, probabilities):
threshold = np.zeros(17)
# iterate over transformations
for t_idx in range(len(transforms)):
# iterate over class labels
t = np.ones(17) * 0.15
selected_preds = probabilities[t_idx]
selected_preds = np.mean(selected_preds, axis=0)
best_thresh = 0.0
best_score = 0.0
for i in range(17):
for r in range(500):
r /= 500
t[i] = r
preds = (selected_preds > t).astype(int)
score = f2_score(labels, preds)
if score > best_score:
best_thresh = r
best_score = score
t[i] = best_thresh
print(
'Transform index {}, score {}, threshold {}, label {}'.format(
t_idx, best_score, best_thresh, i))
print('Transform index {}, threshold {}, score {}'.format(
t_idx, t, best_score))
threshold = threshold + t
threshold = threshold / len(transforms)
return threshold
def find_threshold():
threshold = np.zeros(17)
acc = 0
labels = valid_loader.dataset.labels
pred_files = [f for f in glob.glob('probs/*.txt') if 'blender' in f]
for f in pred_files:
selected_preds = np.loadtxt(f)
t = np.ones(17) * 0.15
# selected_preds = probabilities[t_idx]
# selected_preds = np.mean(selected_preds, axis=0)
best_thresh = 0.0
best_score = 0.0
for i in range(17):
for r in range(500):
r /= 500
t[i] = r
preds = (selected_preds > t).astype(int)
score = f2_score(labels, preds)
if score > best_score:
best_thresh = r
best_score = score
t[i] = best_thresh
threshold = threshold + t
acc += best_score
print('AVG ACC,', acc / len(pred_files))
threshold = threshold / len(pred_files)
return threshold