def eval_classifier(model, data_loader, args, writer=None, epoch=0):
"""Evaluate model on val/test data."""
model.eval()
#model.enable_all_grads()
device = args.device
kbar = pkbar.Kbar(target=len(data_loader), width=25)
gt = []
pred = []
cam = GradCAM(
model=model, target_layer=model.net.layer4[-1],
use_cuda=True if torch.cuda.is_available() else False
)
for step, ex in enumerate(data_loader):
images, _, emotions, _ = ex
images = images.to(device)
pred.append(torch.sigmoid(model(images)).detach().cpu().numpy())
gt.append(emotions.cpu().numpy())
kbar.update(step)
# Log
writer.add_image(
'image_sample',
back2color(unnormalize_imagenet_rgb(images[1], device)),
epoch * len(data_loader) + step
)
for emo_id in torch.nonzero(emotions[1]).reshape(-1):
grayscale_cam = cam(
input_tensor=images[1:2],
target_category=emo_id.item()
)
#grayscale_cam = grayscale_cam[0]
'''
writer.add_image(
'gray_grad_cam_{}'.format(emo_id.item()),
torch.from_numpy(np.uint8(255*grayscale_cam)).unsqueeze(0).repeat(3,1,1),
epoch * len(data_loader) + step
)
'''
heatmap = cv2.cvtColor(
cv2.applyColorMap(np.uint8(255*grayscale_cam), cv2.COLORMAP_JET),
cv2.COLOR_BGR2RGB
)
heatmap = torch.from_numpy(np.float32(heatmap) / 255).to(device)
rgb_img = unnormalize_imagenet_rgb(images[1], device)
rgb_cam_vis = heatmap.permute(2, 0, 1).contiguous() + rgb_img
rgb_cam_vis = rgb_cam_vis / torch.max(rgb_cam_vis).item()
writer.add_image(
'image_grad_cam_{}'.format(emo_id.item()),
back2color(rgb_cam_vis),
epoch * len(data_loader) + step
)
AP = compute_ap(np.concatenate(gt), np.concatenate(pred))
print(f"\nAccuracy: {np.mean(AP)}")
print(AP)
#model.zero_grad()
#model.disable_all_grads()
return np.mean(AP)
def calculate_average_ap_on_batch(images, gt_boxes, gt_classes, gt_masks, pred_boxes, pred_classes, pred_classes_scores, pred_masks):
mask_sum = 0
box_sum = 0
for i in range(len(images)):
mask_ap, box_ap = metrics.compute_ap(gt_boxes[i], gt_classes[i], gt_masks[i], pred_boxes[i], pred_classes[i], pred_classes_scores[i], pred_masks[i])
mask_sum += mask_ap
box_sum += box_ap
return mask_sum, box_sum
def test(fasterrcnn_model):
OutputStore.check_folder(OutputStore.dir_output_test_images)
start_time = time.strftime('%Y/%m/%d %H:%M:%S')
print("-" * 100)
print(f"{start_time} Starting testing the model")
class_data = ClassData()
test_dataloader = PascalVOC2012Dataset.get_data_loader(
Constants.dir_test_images, "test")
fasterrcnn_model.eval() # Set to eval mode for validation
step = 0
tot_MAP = 0
for id, batch in enumerate(test_dataloader):
_, X, img, img_file, y = batch
step += 1
if step % 100 == 0:
curr_time = time.strftime('%Y/%m/%d %H:%M:%S')
print(f"-- {curr_time} step: {step}")
X, y['labels'], y['boxes'] = X.to(Constants.device), y['labels'].to(
Constants.device), y['boxes'].to(Constants.device)
img_file = img_file[0]
img = np.squeeze(img)
# list of images
images = [im for im in X]
targets = []
lab = {
'boxes': y['boxes'].squeeze_(0),
'labels': y['labels'].squeeze_(0)
}
targets.append(lab)
is_bb_degenerate = check_if_target_bbox_degenerate(targets)
if is_bb_degenerate:
continue # Ignore images with degenerate bounding boxes
# avoid empty objects
if len(targets) == 0:
continue
# Get the predictions from the trained model
predictions = fasterrcnn_model(images, targets)
# predictions = predictions.to(Constants.model)
# now compare the predictions with the ground truth values in the targets
buffer = PredictionBuffer(targets, predictions)
MAP, precisions, recalls, overlaps, gt_match, pred_match = compute_ap(
buffer)
class_data.add_matches(buffer, gt_match, pred_match)
# MAP, precisions, recalls, overlaps = compute_ap(predictions, targets)
tot_MAP += MAP
output_annotated_images(predictions, img, img_file)
output_stats_for_images(MAP, precisions, recalls, overlaps, img_file)
class_results_dict = {}
select_class_results_dict = {}
print("Class level precision and recalls")
print("---------------------------------")
for i in range(Hyper.num_classes):
class_name = Hyper.pascal_categories[i]
class_gt_match = class_data.gt_match_dict[i]
class_pred_match = class_data.pred_match_dict[i]
if len(class_gt_match) > 0:
MAP, precisions, recalls = compute_class_ap(
class_gt_match, class_pred_match)
print(f"MAP for {class_name}: {MAP}")
class_results_dict[class_name] = ClassResultsBuffer(
class_name, i, MAP, precisions, recalls)
if class_name in Hyper.plot_categories:
select_class_results_dict[class_name] = class_results_dict[
class_name]
if MAP == 0:
continue
save_class_metrics(class_name, precisions, recalls)
save_combined_class_metrics(select_class_results_dict)
print("---------------------------------\n\n")
ave_MAP = tot_MAP / step
print(f"Average MAP = {ave_MAP}")
print("*** Test run completed ***")
def train(epoch=0):
# convert list to dict
pascal_voc_classes = {}
for id, name in enumerate(Hyper.pascal_categories):
pascal_voc_classes[name] = id
pascal_voc_classes_name = {v: k for k, v in pascal_voc_classes.items()}
print(pascal_voc_classes, Hyper.num_classes)
# Modeling exercise: train fcn on Pascal VOC
train_dataloader = PascalVOC2012Dataset.get_data_loader(
Constants.dir_images, "train")
val_dataloader = PascalVOC2012Dataset.get_data_loader(
Constants.dir_val_images, "val")
#####################################################################
if Constants.load_model:
fasterrcnn_model, fasterrcnn_optimizer = load_checkpoint(epoch)
else:
fasterrcnn_model, fasterrcnn_optimizer = get_model()
print(fasterrcnn_model)
start_time = time.strftime('%Y/%m/%d %H:%M:%S')
print(f"{start_time} Starting epoch: {epoch}")
total_steps = 0
total_loss = 0
length_dataloader = len(train_dataloader)
loss_per_epoch = []
ave_MAP_per_epoch = []
for _ in range(Hyper.total_epochs):
fasterrcnn_model.train()
epoch += 1
epoch_loss = 0
print(f"Starting epoch: {epoch}")
step = 0
for id, batch in enumerate(train_dataloader):
fasterrcnn_optimizer.zero_grad()
_, X, y = batch
total_steps += 1
step += 1
if step % 100 == 0:
curr_time = time.strftime('%Y/%m/%d %H:%M:%S')
print(
f"-- {curr_time} epoch: {epoch} step: {step} loss: {total_loss}"
)
X, y['labels'], y['boxes'] = X.to(
Constants.device), y['labels'].to(
Constants.device), y['boxes'].to(Constants.device)
# list of images
images = [im for im in X]
targets = []
lab = {
'boxes': y['boxes'].squeeze_(0),
'labels': y['labels'].squeeze_(0)
}
# THIS IS IMPORTANT!!!!!
# get rid of the first dimension (batch)
# IF you have >1 images, make another loop
# REPEAT: DO NOT USE BATCH DIMENSION
targets.append(lab)
is_bb_degenerate = check_if_target_bbox_degenerate(targets)
if is_bb_degenerate:
continue # Ignore images with degenerate bounding boxes
# avoid empty objects
if len(targets) == 0:
continue # Ignore if no targets
loss = fasterrcnn_model(images, targets)
total_loss = 0
for k in loss.keys():
total_loss += loss[k]
epoch_loss += total_loss.item()
total_loss.backward()
fasterrcnn_optimizer.step()
epoch_loss = epoch_loss / length_dataloader
loss_per_epoch.append(epoch_loss)
print(f"Loss in epoch {epoch} = {epoch_loss}")
if Constants.save_model:
checkpoint = {
"state_dict": fasterrcnn_model.state_dict(),
"optimizer": fasterrcnn_optimizer.state_dict(),
"epoch": epoch
}
save_checkpoint(checkpoint)
fasterrcnn_model.eval() # Set to eval mode for validation
step = 0
tot_MAP = 0
for id, batch in enumerate(val_dataloader):
_, X, y = batch
step += 1
X, y['labels'], y['boxes'] = X.to(
Constants.device), y['labels'].to(
Constants.device), y['boxes'].to(Constants.device)
# list of images
images = [im for im in X]
targets = []
lab = {
'boxes': y['boxes'].squeeze_(0),
'labels': y['labels'].squeeze_(0)
}
targets.append(lab)
is_bb_degenerate = check_if_target_bbox_degenerate(targets)
if is_bb_degenerate:
continue # Ignore images with degenerate bounding boxes
# avoid empty objects
if len(targets) == 0:
continue
# Get the predictions from the trained model
predictions = fasterrcnn_model(images, targets)
buffer = PredictionBuffer(targets, predictions)
# now compare the predictions with the ground truth values in the targets
MAP, precisions, recalls, overlaps, gt_match, pred_match = compute_ap(
buffer)
#MAP, precisions, recalls, overlaps = compute_ap(predictions, targets)
tot_MAP += MAP
ave_MAP = tot_MAP / step
ave_MAP_per_epoch.append(ave_MAP)
print(f"Average MAP in epoch {epoch} = {ave_MAP} for {step} images")
save_loss_per_epoch_chart(loss_per_epoch)
save_ave_MAP_per_epoch_chart(ave_MAP_per_epoch)
end_time = time.strftime('%Y/%m/%d %H:%M:%S')
print(f"Training end time: {end_time}")
return fasterrcnn_model
def test_ap(self):
ranks, labels = self._gen_metric_data()
res = metrics.compute_ap(ranks, labels)
gt = ((2 / 3 + 1 / 2) / 2 + (1 + 2 / 3) / 2 + (1 + 1) / 2) / 3
self.assertAllCloseAccordingToType(res['AP'], gt)