def get_predictions(samples,
model_name,
ids,
path_save,
file_names,
val=False):
for (img, m, t, p), id in zip(samples, ids):
for i in range(len(ids)):
image = (img[i].detach().cpu().numpy() * 255).transpose(
1, 2, 0).astype(np.uint8)
boxes = p[i]['boxes'].detach().cpu().numpy()
boxes2 = p[i]['boxes']
targets = t[i]['boxes'].detach().cpu().numpy()
label = t[i]['labels'].detach().cpu().numpy()
scores = p[i]['scores']
preds = p[i]['labels']
new_boxes, _, _ = do_nms(boxes2, scores, preds, threshold=0.2)
bmask = get_bbox_mask(mask=m[i], bbox=boxes)
if label[0] == 0:
bmask2 = get_bbox_mask(mask=m[i], bbox=np.array([]))
else:
bmask2 = get_bbox_mask(mask=m[i], bbox=targets)
bmask3 = get_bbox_mask(mask=m[i],
bbox=new_boxes.detach().cpu().numpy())
if val == True:
Image.fromarray(image.astype(np.uint8)).save(
path_save + '/{}_val_{}_img.png'.format(
model_name, file_names[ids[i].numpy()[0]]),
format='PNG')
Image.fromarray(bmask3.astype(np.uint8)).save(
path_save + '/{}_val_{}_prediction.png'.format(
model_name, file_names[ids[i].numpy()[0]]),
format='PNG')
Image.fromarray(bmask2.astype(np.uint8)).save(
path_save + '/{}_val_{}_target.png'.format(
model_name, file_names[ids[i].numpy()[0]]),
format='PNG')
else:
Image.fromarray(image.astype(np.uint8)).save(
path_save + '/{}_train_{}_img.png'.format(
model_name, ids[i].numpy()[0]),
format='PNG')
Image.fromarray(bmask3.astype(np.uint8)).save(
path_save + '/{}_train_{}_prediction.png'.format(
model_name, ids[i].numpy()[0]),
format='PNG')
Image.fromarray(bmask2.astype(np.uint8)).save(
path_save + '/{}_train_{}_target.png'.format(
model_name, ids[i].numpy()[0]),
format='PNG')
def evaluate(model,
model_name,
optim_name,
lr,
layers,
data_loader,
device,
N,
loss_list,
save_folder,
risk=True,
HPC=True,
multi=False,
scale=True,
threshold=0.3):
n_threads = torch.get_num_threads()
torch.set_num_threads(n_threads)
model.eval()
if HPC:
path_save = save_folder
else:
path_save = r'C:\Users\johan\iCloudDrive\DTU\KID\BA\Kode\Predictions_FRCNN'
metric_logger = utils.MetricLogger(delimiter=" ")
header = 'Test:'
num_boxes_val = []
num_boxes_pred = []
i = 0
mIoU = []
mAP = []
mAP2 = []
conf_matrix = {}
conf_matrix["true_positives"] = 0
conf_matrix["false_positives"] = 0
conf_matrix["true_negatives"] = 0
conf_matrix["false_negatives"] = 0
conf_matrix["total_num_defects"] = 0
conf_matrix["good_leather"] = 0
conf_matrix["bad_leather"] = 0
conf_matrix2 = {}
conf_matrix2["true_positives"] = 0
conf_matrix2["false_positives"] = 0
conf_matrix2["true_negatives"] = 0
conf_matrix2["false_negatives"] = 0
conf_matrix2["total_num_defects"] = 0
conf_matrix2["good_leather"] = 0
conf_matrix2["bad_leather"] = 0
with torch.no_grad():
for (image, labels,
masks) in metric_logger.log_every(data_loader, 100, header):
images = list(img.to(device) for img in image)
targets = list(
{k: v.to(device, dtype=torch.long)
for k, v in t.items()} for t in labels)
torch.cuda.synchronize()
model_time = time.time()
outputs = model(images)
outputs = [{k: v.to(device)
for k, v in t.items()} for t in outputs]
model_time = time.time() - model_time
ids = [targets[i]['image_id'].cpu() for i in range(len(targets))]
evaluator_time = time.time()
evaluator_time = time.time() - evaluator_time
for j in range(len(ids)):
if multi:
iou, label_list = iou_multi(
boxes=outputs[j]['boxes'].cpu(),
targets=targets[j]['boxes'].cpu(),
pred=outputs[j]['labels'].cpu(),
labels=targets[j]['labels'].cpu())
df, AP, AP2, c = get_class_iou(
iou_list=iou,
label_list=label_list,
scores=outputs[j]['scores'].cpu(),
target=targets[j]['boxes'].cpu(),
labels=targets[j]['labels'].cpu(),
preds=outputs[j]['labels'].cpu(),
threshold=threshold)
mAP.append(AP)
mAP2.append(AP2)
if N % 50 == 0:
df2, _, _ = get_map2(outputs[j]['boxes'],
targets[j]['boxes'],
outputs[j]['scores'],
outputs[j]['labels'].cpu(),
targets[j]['labels'].cpu(),
iou_list=iou,
threshold=threshold,
print_state=True)
else:
boxes = outputs[j]['boxes'].cpu()
scores = outputs[j]['scores'].cpu()
preds = outputs[j]['labels'].cpu()
new_boxes, new_scores, new_preds = do_nms(boxes,
scores,
preds,
threshold=0.2)
if scale == True:
expand = 42
else:
expand = 21
iou_target, iou_pred = get_iou_targets(
boxes=new_boxes,
targets=targets[j]['boxes'].cpu(),
preds=new_preds,
labels=targets[j]['labels'].cpu(),
image=images[j],
expand=expand)
acc_dict = classifier_metric(iou_target, iou_pred,
new_scores,
targets[j]['boxes'].cpu(),
targets[j]['labels'].cpu())
conf_matrix["true_positives"] += acc_dict["Detected"]
conf_matrix["false_negatives"] += acc_dict[
"Defects"] - acc_dict["Detected"]
conf_matrix["false_positives"] += acc_dict["FP"]
conf_matrix["total_num_defects"] += acc_dict["Defects"]
if acc_dict["Defects"] == 0:
conf_matrix["good_leather"] += 1
if acc_dict["FP"] == 0:
conf_matrix["true_negatives"] += 1
else:
conf_matrix["bad_leather"] += 1
iou_target2, iou_pred2 = get_iou_targets(
boxes=boxes,
targets=targets[j]['boxes'].cpu(),
preds=preds,
labels=targets[j]['labels'].cpu(),
image=images[j],
expand=expand)
acc_dict2 = classifier_metric(iou_target2, iou_pred2,
scores,
targets[j]['boxes'].cpu(),
targets[j]['labels'].cpu())
conf_matrix2["true_positives"] += acc_dict2["Detected"]
conf_matrix2["false_negatives"] += acc_dict2[
"Defects"] - acc_dict2["Detected"]
conf_matrix2["false_positives"] += acc_dict2["FP"]
conf_matrix2["total_num_defects"] += acc_dict2["Defects"]
if acc_dict2["Defects"] == 0:
conf_matrix2["good_leather"] += 1
if acc_dict2["FP"] == 0:
conf_matrix2["true_negatives"] += 1
else:
conf_matrix2["bad_leather"] += 1
iou, _, _ = get_iou2(boxes=new_boxes,
targets=targets[j]['boxes'].cpu(),
pred=new_preds,
labels=targets[j]['labels'].cpu())
df, _, AP = get_map2(new_boxes,
targets[j]['boxes'],
new_scores,
new_preds,
targets[j]['labels'].cpu(),
iou_list=iou,
threshold=threshold)
iou2, _, _ = get_iou2(boxes=outputs[j]['boxes'].cpu(),
targets=targets[j]['boxes'].cpu(),
pred=outputs[j]['labels'].cpu(),
labels=targets[j]['labels'].cpu())
df2, _, AP2 = get_map2(outputs[j]['boxes'],
targets[j]['boxes'],
outputs[j]['scores'],
outputs[j]['labels'].cpu(),
targets[j]['labels'].cpu(),
iou_list=iou2,
threshold=threshold)
mAP.append(AP)
mAP2.append(AP2)
IoU = mask_iou(boxes=new_boxes,
mask=masks[j],
targets=targets[j]['boxes'].cpu())
mIoU.append(IoU[1])
samples = []
num_boxes_val.append(
np.mean([len(targets[i]['boxes']) for i in range(len(ids))]))
num_boxes_pred.append(
np.mean([len(outputs[i]['boxes']) for i in range(len(ids))]))
samples.append((images, masks, targets, outputs))
if N % 40 == 0:
metric_logger.update(model_time=model_time,
evaluator_time=evaluator_time)
if risk == True:
model.train()
images = list(
img.to(device, dtype=torch.float32) for img in image)
targets = list({k: v.to(device)
for k, v in t.items()} for t in labels)
loss_dict = model(images, targets)
# reduce losses over all GPUs for logging purposes
loss_dict_reduced = utils.reduce_dict(loss_dict)
losses_reduced = sum(loss
for loss in loss_dict_reduced.values())
loss_value = losses_reduced.item()
loss_list.append(loss_value)
model.eval()
i += 1
# gather the stats from all processes
metric_logger.synchronize_between_processes()
if HPC:
if N % 40 == 0:
print("Averaged stats:", metric_logger)
print(
"mean Average Precision for epoch {} with nms: ".format(N),
np.mean(mAP))
print("mean Average Precision without nms {}: ".format(N),
np.mean(mAP2))
print("mean IoU with nms: ", np.mean(mIoU))
print("TP: ", conf_matrix["true_positives"])
print("FP: ", conf_matrix["false_positives"])
print("TN: ", conf_matrix["true_negatives"])
print("FN: ", conf_matrix["false_negatives"])
print("Total number of defects: ",
conf_matrix["total_num_defects"])
print("Images with good leather: ",
conf_matrix["good_leather"])
print("Images with bad leather: ", conf_matrix["bad_leather"])
else:
print("Averaged stats:", metric_logger)
print("mean Average Precision for epoch {} with nms: ".format(N),
np.mean(mAP))
print("mean Average Precision without nms {}: ".format(N),
np.mean(mAP2))
print("mean IoU with nms: ", np.mean(mIoU))
print("TP: ", conf_matrix["true_positives"])
print("FP: ", conf_matrix["false_positives"])
print("TN: ", conf_matrix["true_negatives"])
print("FN: ", conf_matrix["false_negatives"])
print("Total number of defects: ",
conf_matrix["total_num_defects"])
print("Images with good leather: ", conf_matrix["good_leather"])
print("Images with bad leather: ", conf_matrix["bad_leather"])
return np.mean(mAP), np.mean(mAP2), np.mean(loss_list), np.mean(
np.array(num_boxes_pred)), np.mean(np.array(
num_boxes_val)), conf_matrix, conf_matrix2, np.nanmean(mIoU)
target_tif = []
image_tif = []
print("Loop over: ", split_x_y[0], "and ", split_x_y[1])
pred_counter = 0
for i in range(split_x_y[0]):
if i % 20 == 0:
print("i ", i)
pred_stack = []
img_stack = []
for j in range(split_x_y[1]):
label = Image.fromarray(np.zeros((1,split_imgs[i * split_x_y[1] + j].size), dtype=np.uint8))
outputs, size = output(model,array=split_imgs[i * split_x_y[1] + j],device=device)
boxes = outputs[0]['boxes'].cpu()
scores = outputs[0]['scores'].cpu()
preds = outputs[0]['labels'].cpu()
new_boxes, new_scores, _ = do_nms(boxes.detach(), scores.detach(), preds.detach(), threshold=0.2)
pred = create_mask_from_bbox(new_boxes.detach().cpu().numpy(),size[0])
pred, num_boxes, mod_boxes = adjust_bbox_tif(new_boxes.detach().cpu().numpy(),adjust=padding,size=size[0])
pred = fill_bbox(mod_boxes,np.zeros((np.shape(pred))))
pred_counter += num_boxes
if do_pad:
pred = pred[50:-50, 50:-50]
if isinstance(pred_stack, list):
pred_stack = pred
if do_pad:
img_stack = split_imgs[i * split_x_y[1] + j][50:-50,50:-50,:]
else:
img_stack = split_imgs[i * split_x_y[1] + j]
else:
pred_stack = np.hstack((pred_stack, pred))
if do_pad:
def validate(model,
model_name,
data_loader,
device,
path_save,
bbox_type,
file_names,
resize=True,
val=True,
bbox=False,
threshold=0.3):
if bbox == False:
if val == True:
path_save = os.path.join(path_save, 'val')
else:
path_save = os.path.join(path_save, 'train')
else:
path_save = os.path.join(path_save, bbox_type)
i = 0
if resize:
expand = 42
else:
expand = 84
conf_matrix = {}
conf_matrix["true_positives"] = 0
conf_matrix["false_positives"] = 0
conf_matrix["true_negatives"] = 0
conf_matrix["false_negatives"] = 0
conf_matrix["total_num_defects"] = 0
conf_matrix["good_leather"] = 0
conf_matrix["bad_leather"] = 0
conf_matrix2 = {}
conf_matrix2["true_positives"] = 0
conf_matrix2["false_positives"] = 0
conf_matrix2["true_negatives"] = 0
conf_matrix2["false_negatives"] = 0
conf_matrix2["total_num_defects"] = 0
conf_matrix2["good_leather"] = 0
conf_matrix2["bad_leather"] = 0
mAP = []
mAP2 = []
with torch.no_grad():
for (image, labels, masks) in data_loader:
images = list(img.to(device) for img in image)
targets = list(
{k: v.to(device, dtype=torch.long)
for k, v in t.items()} for t in labels)
torch.cuda.synchronize()
model_time = time.time()
outputs = model(images)
outputs = [{k: v.to(device)
for k, v in t.items()} for t in outputs]
model_time = time.time() - model_time
ids = [targets[i]['image_id'].cpu() for i in range(len(targets))]
evaluator_time = time.time()
evaluator_time = time.time() - evaluator_time
for j in range(len(ids)):
boxes = outputs[j]['boxes'].cpu()
scores = outputs[j]['scores'].cpu()
preds = outputs[j]['labels'].cpu()
#new_boxes, new_scores, new_labels = get_non_maximum_supression(boxes, scores, preds, iou_threshold=0.2)
new_boxes, new_scores, new_preds = do_nms(boxes,
scores,
preds,
threshold=0.2)
iou_target, iou_pred = get_iou_targets(
boxes=new_boxes,
targets=targets[j]['boxes'].cpu(),
preds=new_preds,
labels=targets[j]['labels'].cpu(),
image=images[j],
expand=expand)
iou_target2, iou_pred2 = get_iou_targets(
boxes=boxes,
targets=targets[j]['boxes'].cpu(),
preds=preds,
labels=targets[j]['labels'].cpu(),
image=images[j],
expand=expand)
acc_dict = classifier_metric(iou_target, iou_pred, new_scores,
targets[j]['boxes'].cpu(),
targets[j]['labels'].cpu())
conf_matrix["true_positives"] += acc_dict["Detected"]
conf_matrix["false_negatives"] += acc_dict[
"Defects"] - acc_dict["Detected"]
conf_matrix["false_positives"] += acc_dict["FP"]
conf_matrix["total_num_defects"] += acc_dict["Defects"]
if acc_dict["Defects"] == 0:
conf_matrix["good_leather"] += 1
if acc_dict["FP"] == 0:
conf_matrix["true_negatives"] += 1
else:
conf_matrix["bad_leather"] += 1
acc_dict2 = classifier_metric(iou_target2, iou_pred2, scores,
targets[j]['boxes'].cpu(),
targets[j]['labels'].cpu())
conf_matrix2["true_positives"] += acc_dict2["Detected"]
conf_matrix2["false_negatives"] += acc_dict2[
"Defects"] - acc_dict2["Detected"]
conf_matrix2["false_positives"] += acc_dict2["FP"]
conf_matrix2["total_num_defects"] += acc_dict2["Defects"]
if acc_dict2["Defects"] == 0:
conf_matrix2["good_leather"] += 1
if acc_dict2["FP"] == 0:
conf_matrix2["true_negatives"] += 1
else:
conf_matrix2["bad_leather"] += 1
iou, index, selected_iou = get_iou2(
boxes=outputs[j]['boxes'].cpu(),
targets=targets[j]['boxes'].cpu(),
pred=outputs[j]['labels'].cpu(),
labels=targets[j]['labels'].cpu())
df, _, AP = get_map2(outputs[j]['boxes'],
targets[j]['boxes'],
outputs[j]['scores'],
outputs[j]['labels'].cpu(),
targets[j]['labels'].cpu(),
iou_list=iou,
threshold=threshold)
iou2, _, _ = get_iou2(boxes=new_boxes,
targets=targets[j]['boxes'].cpu(),
pred=new_preds,
labels=targets[j]['labels'].cpu())
df2, _, AP2 = get_map2(new_boxes,
targets[j]['boxes'],
new_scores,
new_preds,
targets[j]['labels'].cpu(),
iou_list=iou2,
threshold=threshold)
mAP.append(AP)
mAP2.append(AP2)
samples = []
samples.append((images, expanded_targets, targets, outputs))
get_predictions(samples,
model_name,
ids,
path_save=path_save,
file_names=file_names,
val=val)
i += 1
return np.mean(mAP), np.mean(mAP2), conf_matrix, conf_matrix2