def test_embeddings(self):
# setup model.
model = models.resnet50(pretrained=True)
model.eval()
# loop over dataset.
for i, (images, target) in enumerate(self.data_loader):
print(f'\n{i}/{len(self.data_loader)} ..')
# format rgb.
image = np.squeeze(np.array(images))
image, target = arl_affpose_dataset_utils.format_target_data(
image, target)
# get class label.
obj_mask = target['obj_mask']
labels = np.unique(obj_mask)[1:]
with torch.no_grad():
images = images.view(config.BATCH_SIZE, image.shape[2],
image.shape[0],
image.shape[1]).type(torch.float32)
outputs = model(images)
current_outputs = outputs.cpu().numpy()
features = np.concatenate((outputs, current_outputs))
tsne = TSNE(n_components=2).fit_transform(features)
def test_class_distributions(self):
num_obj_ids = np.zeros(shape=(11 + 1))
num_aff_ids = np.zeros(shape=(9 + 1))
# loop over dataset.
for i, (image, target) in enumerate(self.data_loader):
if i % 10 == 0:
print(f'{i}/{len(self.data_loader)} ..')
# format data.
image = np.squeeze(np.array(image)).transpose(1, 2, 0)
image = np.array(image * (2**8 - 1),
dtype=np.uint8).reshape(config.ARL_CROP_SIZE[0],
config.ARL_CROP_SIZE[1],
-1)
image, target = arl_affpose_dataset_utils.format_target_data(
image, target)
# obj_ids.
obj_ids = target['obj_ids']
for obj_id in obj_ids:
num_obj_ids[0] += 1
num_obj_ids[obj_id] += 1
# aff_ids.
aff_ids = target['aff_ids']
for aff_id in aff_ids:
num_aff_ids[0] += 1
num_aff_ids[aff_id] += 1
print(f'num_obj_ids: {num_obj_ids}')
print(f'num_aff_ids: {num_aff_ids}')
def test_maskrcnn_dataloader(self):
print('\nVisualizing Ground Truth Data for MaskRCNN ..')
# loop over dataset.
for i, (image, target) in enumerate(self.data_loader):
print(f'{i}/{len(self.data_loader)} ..')
# format data.
image = np.squeeze(np.array(image)).transpose(1, 2, 0)
image = np.array(image * (2**8 - 1),
dtype=np.uint8).reshape(config.ARL_CROP_SIZE[0],
config.ARL_CROP_SIZE[1],
-1)
target = arl_affpose_dataset_utils.format_target_data(
image, target)
# Bounding Box.
bbox_img = arl_affpose_dataset_utils.draw_bbox_on_img(
image=image,
obj_ids=target['obj_ids'],
boxes=target['obj_boxes'],
)
# Original Segmentation Mask.
color_mask = arl_affpose_dataset_utils.colorize_obj_mask(
target['obj_mask'])
color_mask = cv2.addWeighted(bbox_img, 0.35, color_mask, 0.65, 0)
# Binary Masks.
binary_mask = arl_affpose_dataset_utils.get_segmentation_masks(
image=image,
obj_ids=target['obj_ids'],
binary_masks=target['obj_binary_masks'],
)
color_binary_mask = arl_affpose_dataset_utils.colorize_obj_mask(
binary_mask)
color_binary_mask = cv2.addWeighted(bbox_img, 0.35,
color_binary_mask, 0.65, 0)
# print object and affordance class names.
arl_affpose_dataset_utils.print_class_obj_names(target['obj_ids'])
# show plots.
cv2.imshow('rgb', cv2.cvtColor(image, cv2.COLOR_BGR2RGB))
cv2.imshow('bbox', cv2.cvtColor(bbox_img, cv2.COLOR_BGR2RGB))
cv2.imshow('mask', cv2.cvtColor(color_mask, cv2.COLOR_BGR2RGB))
cv2.imshow('binary_mask',
cv2.cvtColor(color_binary_mask, cv2.COLOR_BGR2RGB))
cv2.waitKey(0)
def test_rgbd_distribution(self):
# init stats.
rgb_mean, rgb_std = 0, 0
depth_mean, depth_std = 0, 0
distance_to_object_mean, distance_to_object_std = 0, 0
nb_bins = int(2**8)
count_r = np.zeros(nb_bins)
count_g = np.zeros(nb_bins)
count_b = np.zeros(nb_bins)
count_d = np.zeros(nb_bins)
# loop over dataset.
for i, (image, target) in enumerate(self.data_loader):
if i % 10 == 0:
print(f'{i}/{len(self.data_loader)} ..')
# format data.
image = np.squeeze(np.array(image)).transpose(1, 2, 0)
image = np.array(image * (2**8 - 1),
dtype=np.uint8).reshape(config.ARL_CROP_SIZE[0],
config.ARL_CROP_SIZE[1],
-1)
image, target = arl_affpose_dataset_utils.format_target_data(
image, target)
# get depth.
depth_8bit = target['depth_8bit']
depth_16bit = target['depth_16bit']
masked_depth_16bit = target['masked_depth_16bit']
# set depth for stats.
depth = depth_8bit
# mean and std
# rgb
img_stats = image
img_stats = img_stats.reshape(3, -1)
rgb_mean += np.mean(img_stats, axis=1)
rgb_std += np.std(img_stats, axis=1)
# getting nonzero mean depth.
img_stats = image
img_stats = img_stats.reshape(1, -1)
depth_mean += np.mean(img_stats, axis=1)
depth_std += np.std(img_stats, axis=1)
# getting nonzero mean depth.
masked_depth_16bit_nan = np.where(masked_depth_16bit != 0,
masked_depth_16bit, np.nan)
img_stats = masked_depth_16bit_nan.reshape(1, -1)
distance_to_object_mean += np.nanmean(img_stats, axis=1)
distance_to_object_std += np.nanstd(img_stats, axis=1)
# histogram.
# rgb
hist_r = np.histogram(image[0], bins=nb_bins, range=[0, 255])
hist_g = np.histogram(image[1], bins=nb_bins, range=[0, 255])
hist_b = np.histogram(image[2], bins=nb_bins, range=[0, 255])
count_r += hist_r[0]
count_g += hist_g[0]
count_b += hist_b[0]
# depth
hist_d = np.histogram(depth, bins=nb_bins, range=[0, 255])
count_d += hist_d[0]
# get stats.
rgb_mean /= i
rgb_std /= i
print(f'\nRGB: mean:{rgb_mean}\nstd:{rgb_std}')
depth_mean /= i
depth_std /= i
print(f'Depth: mean:{depth_mean}\nstd:{depth_std}')
distance_to_object_mean /= i
distance_to_object_std /= i
print(
f'Distance to Object: mean:{distance_to_object_mean}\nstd:{distance_to_object_std}'
)
# plot histograms.
plt.close()
bins = hist_r[1]
# rgb
plt.figure(figsize=(12, 6))
plt.bar(hist_r[1][:-1], count_r, color='r', label='Red', alpha=0.33)
plt.axvline(x=rgb_mean[0], color='r', ls='--')
plt.bar(hist_g[1][:-1], count_g, color='g', label='Green', alpha=0.33)
plt.axvline(x=rgb_mean[1], color='g', ls='--')
plt.bar(hist_b[1][:-1], count_b, color='b', label='Blue', alpha=0.33)
plt.axvline(x=rgb_mean[2], color='b', ls='--')
plt.legend(bbox_to_anchor=(1.0, 1), loc='upper left')
plt.show()
# depth
plt.figure(figsize=(12, 6))
plt.bar(x=hist_d[1][:-1],
height=count_d,
color='k',
label='depth',
alpha=0.33)
plt.axvline(x=depth_mean, color='k', ls='--')
plt.axvline(x=rgb_mean[0], color='r', ls='--')
plt.axvline(x=rgb_mean[1], color='g', ls='--')
plt.axvline(x=rgb_mean[2], color='b', ls='--')
plt.legend(bbox_to_anchor=(1.0, 1), loc='upper left')
plt.show()
def affnet_eval_arl_affpose(model, test_loader):
print('\nevaluating AffNet ..')
# set the model to eval to disable batchnorm.
model.eval()
# Init folders.
if not os.path.exists(config.ARL_TEST_SAVE_FOLDER):
os.makedirs(config.ARL_TEST_SAVE_FOLDER)
gt_pred_images = glob.glob(config.ARL_TEST_SAVE_FOLDER + '*')
for images in gt_pred_images:
os.remove(images)
APs = []
gt_obj_ids_list, pred_obj_ids_list = [], []
for image_idx, (images, targets) in enumerate(test_loader):
image, target = copy.deepcopy(images), copy.deepcopy(targets)
images = list(image.to(config.DEVICE) for image in images)
with torch.no_grad():
outputs = model(images)
outputs = [{k: v.to(config.CPU_DEVICE)
for k, v in t.items()} for t in outputs]
# Formatting input.
image = image[0]
image = image.to(config.CPU_DEVICE)
image = np.squeeze(np.array(image)).transpose(1, 2, 0)
image = np.array(image * (2**8 - 1), dtype=np.uint8)
H, W, C = image.shape
# Formatting targets.
target = target[0]
target = {k: v.to(config.CPU_DEVICE) for k, v in target.items()}
target = arl_affpose_dataset_utils.format_target_data(
image.copy(), target.copy())
gt_obj_ids = np.array(target['obj_ids'], dtype=np.int32).flatten()
gt_obj_boxes = np.array(target['obj_boxes'],
dtype=np.int32).reshape(-1, 4)
gt_obj_binary_masks = np.array(target['obj_binary_masks'],
dtype=np.uint8).reshape(-1, H, W)
# Formatting Output.
outputs = outputs.pop()
outputs = affnet_format_outputs(image.copy(), outputs.copy())
outputs = affnet_threshold_outputs(image.copy(), outputs.copy())
outputs = maskrcnn_match_pred_to_gt(image.copy(), target.copy(),
outputs.copy())
scores = np.array(outputs['scores'], dtype=np.float32).flatten()
obj_ids = np.array(outputs['obj_ids'], dtype=np.int32).flatten()
obj_boxes = np.array(outputs['obj_boxes'],
dtype=np.int32).reshape(-1, 4)
obj_binary_masks = np.array(outputs['obj_binary_masks'],
dtype=np.uint8).reshape(-1, H, W)
aff_scores = np.array(outputs['aff_scores'],
dtype=np.float32).flatten()
obj_part_ids = np.array(outputs['obj_part_ids'],
dtype=np.int32).flatten()
aff_ids = np.array(outputs['aff_ids'], dtype=np.int32).flatten()
aff_binary_masks = np.array(outputs['aff_binary_masks'],
dtype=np.uint8).reshape(-1, H, W)
# confusion matrix.
gt_obj_ids_list.extend(gt_obj_ids.tolist())
pred_obj_ids_list.extend(obj_ids.tolist())
# get average precision.
AP = compute_ap_range(
gt_class_id=gt_obj_ids,
gt_box=gt_obj_boxes,
gt_mask=gt_obj_binary_masks.reshape(H, W, -1),
pred_score=scores,
pred_class_id=obj_ids,
pred_box=obj_boxes,
pred_mask=obj_binary_masks.reshape(H, W, -1),
verbose=False,
)
APs.append(AP)
# get aff masks.
aff_mask = arl_affpose_dataset_utils.get_segmentation_masks(
image=image,
obj_ids=aff_ids,
binary_masks=aff_binary_masks,
)
gt_name = config.ARL_TEST_SAVE_FOLDER + str(
image_idx) + config.TEST_GT_EXT
cv2.imwrite(gt_name, target['aff_mask'])
pred_name = config.ARL_TEST_SAVE_FOLDER + str(
image_idx) + config.TEST_PRED_EXT
cv2.imwrite(pred_name, aff_mask)
# Confusion Matrix.
cm = sklearn_confusion_matrix(y_true=gt_obj_ids_list,
y_pred=pred_obj_ids_list)
print(f'\n{cm}')
# mAP
mAP = np.mean(APs)
print(f'\nmAP: {mAP:.5f}')
# getting Fwb
os.chdir(config.MATLAB_SCRIPTS_DIR)
import matlab.engine
eng = matlab.engine.start_matlab()
Fwb = eng.evaluate_arl_affpose_affnet(config.ARL_TEST_SAVE_FOLDER,
nargout=1)
os.chdir(config.ROOT_DIR_PATH)
model.train()
return model, mAP, Fwb
def main():
if SAVE_AND_EVAL_PRED:
# Init folders
print('\neval in .. {}'.format(config.ARL_OBJ_EVAL_SAVE_FOLDER))
if not os.path.exists(config.ARL_OBJ_EVAL_SAVE_FOLDER):
os.makedirs(config.ARL_OBJ_EVAL_SAVE_FOLDER)
gt_pred_images = glob.glob(config.ARL_OBJ_EVAL_SAVE_FOLDER + '*')
for images in gt_pred_images:
os.remove(images)
# Load the Model.
print()
# Compare Pytorch-Simple-MaskRCNN. with Torchvision MaskRCNN.
model = model_utils.get_model_instance_segmentation(
pretrained=config.IS_PRETRAINED, num_classes=config.NUM_CLASSES)
model.to(config.DEVICE)
# Load saved weights.
print(
f"\nrestoring pre-trained MaskRCNN weights: {config.RESTORE_ARL_TORCHVISION_MASKRCNN_WEIGHTS} .. "
)
checkpoint = torch.load(config.RESTORE_ARL_TORCHVISION_MASKRCNN_WEIGHTS,
map_location=config.DEVICE)
model.load_state_dict(checkpoint["model"])
model.eval()
# Load the dataset.
test_loader = arl_affpose_dataset_loaders.load_arl_affpose_eval_datasets(
random_images=RANDOM_IMAGES,
num_random=NUM_RANDOM,
shuffle_images=SHUFFLE_IMAGES)
# run the predictions.
APs = []
for image_idx, (images, targets) in enumerate(test_loader):
print(f'\nImage:{image_idx}/{len(test_loader)}')
image, target = copy.deepcopy(images), copy.deepcopy(targets)
images = list(image.to(config.DEVICE) for image in images)
with torch.no_grad():
outputs = model(images)
outputs = [{k: v.to(config.CPU_DEVICE)
for k, v in t.items()} for t in outputs]
# Formatting input.
image = image[0]
image = image.to(config.CPU_DEVICE)
image = np.squeeze(np.array(image)).transpose(1, 2, 0)
image = np.array(image * (2**8 - 1), dtype=np.uint8)
H, W, C = image.shape
target = target[0]
target = {k: v.to(config.CPU_DEVICE) for k, v in target.items()}
image, target = arl_affpose_dataset_utils.format_target_data(
image, target)
# format outputs by most confident.
outputs = outputs.pop()
outputs['obj_ids'] = outputs['labels']
outputs['obj_boxes'] = outputs['boxes']
outputs['obj_binary_masks'] = outputs['masks']
image, outputs = arl_affpose_dataset_utils.format_outputs(
image, outputs)
scores = np.array(outputs['scores'], dtype=np.float32).flatten()
obj_ids = np.array(outputs['obj_ids'], dtype=np.int32).flatten()
obj_boxes = np.array(outputs['obj_boxes'],
dtype=np.int32).reshape(-1, 4)
obj_binary_masks = np.array(outputs['obj_binary_masks'],
dtype=np.uint8).reshape(-1, H, W)
# match gt to pred.
num_gt, num_pred = len(target['obj_ids']), len(outputs['obj_ids'])
target, outputs = eval_utils.get_matched_predictions(
image.copy(), target, outputs)
gt_obj_ids = np.array(target['obj_ids'], dtype=np.int32).flatten()
gt_obj_boxes = np.array(target['obj_boxes'],
dtype=np.int32).reshape(-1, 4)
gt_obj_binary_masks = np.array(target['obj_binary_masks'],
dtype=np.uint8).reshape(-1, H, W)
# Quantify pred.
print(f"num gt: {num_gt},\t num preds: {num_pred}")
for gt_idx, pred_idx in zip(range(len(gt_obj_ids)),
range(len(obj_ids))):
gt_obj_name = "{:<15}".format(
arl_affpose_dataset_utils.map_obj_id_to_name(
gt_obj_ids[gt_idx]))
pred_obj_name = "{:<15}".format(
arl_affpose_dataset_utils.map_obj_id_to_name(
obj_ids[pred_idx]))
score = scores[pred_idx]
bbox_iou = eval_utils.get_iou(obj_boxes[pred_idx],
gt_obj_boxes[gt_idx])
if gt_obj_name == pred_obj_name and score > config.OBJ_CONFIDENCE_THRESHOLD:
detection = ':) -> [TP]'
elif gt_obj_name != pred_obj_name and score > config.OBJ_CONFIDENCE_THRESHOLD:
detection = 'X -> [FP]'
elif gt_obj_name == pred_obj_name and score < config.OBJ_CONFIDENCE_THRESHOLD:
detection = 'X -> [FN]'
else:
detection = 'X -> [TN]'
print(
f'{detection}\t'
f'Pred: {pred_obj_name}'
f'GT: {gt_obj_name}',
f'Score: {score:.3f},\t\t',
f'IoU: {bbox_iou:.3f},',
)
# get average precision.
print()
AP = eval_utils.compute_ap_range(
gt_class_id=gt_obj_ids,
gt_box=gt_obj_boxes,
gt_mask=gt_obj_binary_masks.reshape(H, W, -1),
pred_score=scores,
pred_class_id=obj_ids,
pred_box=obj_boxes,
pred_mask=obj_binary_masks.reshape(H, W, -1),
verbose=False,
)
APs.append(AP)
# # visualize all bbox predictions.
pred_bbox_img = arl_affpose_dataset_utils.draw_bbox_on_img(
image=image, scores=scores, obj_ids=obj_ids, boxes=obj_boxes)
# threshold outputs for mask.
image, outputs = arl_affpose_dataset_utils.threshold_outputs(
image, outputs)
scores = np.array(outputs['scores'], dtype=np.float).flatten()
obj_ids = np.array(outputs['obj_ids'], dtype=np.int32).flatten()
obj_boxes = np.array(outputs['obj_boxes'],
dtype=np.int32).reshape(-1, 4)
obj_binary_masks = np.array(outputs['obj_binary_masks'],
dtype=np.uint8).reshape(-1, H, W)
# visualize all bbox predictions.
bbox_img = arl_affpose_dataset_utils.draw_bbox_on_img(image=image,
scores=scores,
obj_ids=obj_ids,
boxes=obj_boxes)
# only visualize "good" masks.
pred_obj_mask = arl_affpose_dataset_utils.get_segmentation_masks(
image=image, obj_ids=obj_ids, binary_masks=obj_binary_masks)
color_mask = arl_affpose_dataset_utils.colorize_obj_mask(pred_obj_mask)
color_mask = cv2.addWeighted(bbox_img, 0.5, color_mask, 0.5, 0)
if SAVE_AND_EVAL_PRED:
# saving predictions.
_image_idx = target["image_id"].detach().numpy()[0]
_image_idx = str(1000000 + _image_idx)[1:]
gt_name = config.ARL_OBJ_EVAL_SAVE_FOLDER + _image_idx + config.TEST_GT_EXT
pred_name = config.ARL_OBJ_EVAL_SAVE_FOLDER + _image_idx + config.TEST_PRED_EXT
cv2.imwrite(gt_name, target['obj_mask'])
cv2.imwrite(pred_name, pred_obj_mask)
if SHOW_IMAGES:
cv2.imshow('pred_bbox',
cv2.cvtColor(pred_bbox_img, cv2.COLOR_BGR2RGB))
cv2.imshow('pred_mask', cv2.cvtColor(color_mask,
cv2.COLOR_BGR2RGB))
cv2.waitKey(0)
# Precision and Recall
print("mAP @0.5-0.95: over {} test images is {:.3f}".format(
len(APs), np.mean(APs)))
if SAVE_AND_EVAL_PRED:
print()
# getting FwB.
os.chdir(config.MATLAB_SCRIPTS_DIR)
import matlab.engine
eng = matlab.engine.start_matlab()
Fwb = eng.evaluate_arl_affpose_maskrcnn(
config.ARL_OBJ_EVAL_SAVE_FOLDER, nargout=1)
os.chdir(config.ROOT_DIR_PATH)
def main():
# if SAVE_AND_EVAL_PRED:
# # Init folders
# print('\neval in .. {}'.format(config.ARL_AFF_EVAL_SAVE_FOLDER))
#
# if not os.path.exists(config.ARL_AFF_EVAL_SAVE_FOLDER):
# os.makedirs(config.ARL_AFF_EVAL_SAVE_FOLDER)
#
# gt_pred_images = glob.glob(config.ARL_AFF_EVAL_SAVE_FOLDER + '*')
# for images in gt_pred_images:
# os.remove(images)
# Load the Model.
print()
model = affnet.ResNetAffNet(pretrained=config.IS_PRETRAINED,
num_classes=config.NUM_CLASSES)
model.to(config.DEVICE)
# Load saved weights.
print(
f"\nrestoring pre-trained MaskRCNN weights: {config.RESTORE_ARL_AFFNET_WEIGHTS} .. "
)
checkpoint = torch.load(config.RESTORE_ARL_AFFNET_WEIGHTS,
map_location=config.DEVICE)
model.load_state_dict(checkpoint["model"])
model.eval()
# Load the dataset.
test_loader = arl_affpose_dataset_loaders.load_arl_affpose_eval_datasets(
random_images=RANDOM_IMAGES,
num_random=NUM_RANDOM,
shuffle_images=SHUFFLE_IMAGES)
# run the predictions.
APs = []
gt_obj_ids_list, pred_obj_ids_list = [], []
for image_idx, (images, targets) in enumerate(test_loader):
print(f'\nImage:{image_idx+1}/{len(test_loader)}')
image, target = copy.deepcopy(images), copy.deepcopy(targets)
images = list(image.to(config.DEVICE) for image in images)
with torch.no_grad():
outputs = model(images)
outputs = [{k: v.to(config.CPU_DEVICE)
for k, v in t.items()} for t in outputs]
# Formatting input.
image = image[0]
image = image.to(config.CPU_DEVICE)
image = np.squeeze(np.array(image)).transpose(1, 2, 0)
image = np.array(image * (2**8 - 1), dtype=np.uint8)
H, W, C = image.shape
# Formatting targets.
target = target[0]
target = {k: v.to(config.CPU_DEVICE) for k, v in target.items()}
target = arl_affpose_dataset_utils.format_target_data(
image.copy(), target.copy())
gt_obj_ids = np.array(target['obj_ids'], dtype=np.int32).flatten()
gt_obj_boxes = np.array(target['obj_boxes'],
dtype=np.int32).reshape(-1, 4)
gt_obj_binary_masks = np.array(target['obj_binary_masks'],
dtype=np.uint8).reshape(-1, H, W)
# Formatting Output.
outputs = outputs.pop()
outputs = eval_utils.affnet_format_outputs(image.copy(),
outputs.copy())
outputs = eval_utils.affnet_threshold_outputs(image.copy(),
outputs.copy())
outputs = eval_utils.maskrcnn_match_pred_to_gt(image.copy(),
target.copy(),
outputs.copy())
scores = np.array(outputs['scores'], dtype=np.float32).flatten()
obj_ids = np.array(outputs['obj_ids'], dtype=np.int32).flatten()
obj_boxes = np.array(outputs['obj_boxes'],
dtype=np.int32).reshape(-1, 4)
obj_binary_masks = np.array(outputs['obj_binary_masks'],
dtype=np.uint8).reshape(-1, H, W)
aff_scores = np.array(outputs['aff_scores'],
dtype=np.float32).flatten()
obj_part_ids = np.array(outputs['obj_part_ids'],
dtype=np.int32).flatten()
aff_ids = np.array(outputs['aff_ids'], dtype=np.int32).flatten()
aff_binary_masks = np.array(outputs['aff_binary_masks'],
dtype=np.uint8).reshape(-1, H, W)
# confusion matrix.
gt_obj_ids_list.extend(gt_obj_ids.tolist())
pred_obj_ids_list.extend(obj_ids.tolist())
# for obj_binary_mask, gt_obj_binary_mask in zip(obj_binary_masks, gt_obj_binary_masks):
# cv2.imshow('obj_binary_mask', obj_binary_mask*20)
# cv2.imshow('gt_obj_binary_mask', gt_obj_binary_mask*20)
# cv2.waitKey(0)
# get average precision.
AP = eval_utils.compute_ap_range(
gt_class_id=gt_obj_ids,
gt_box=gt_obj_boxes,
gt_mask=gt_obj_binary_masks.reshape(H, W, -1),
pred_score=scores,
pred_class_id=obj_ids,
pred_box=obj_boxes,
pred_mask=obj_binary_masks.reshape(H, W, -1),
verbose=False,
)
APs.append(AP)
# print outputs.
for gt_idx, pred_idx in zip(range(len(gt_obj_ids)),
range(len(obj_ids))):
gt_obj_name = "{:<15}".format(
arl_affpose_dataset_utils.map_obj_id_to_name(
gt_obj_ids[gt_idx]))
pred_obj_name = "{:<15}".format(
arl_affpose_dataset_utils.map_obj_id_to_name(
obj_ids[pred_idx]))
score = scores[pred_idx]
bbox_iou = eval_utils.get_iou(obj_boxes[pred_idx],
gt_obj_boxes[gt_idx])
print(
f'GT: {gt_obj_name}',
f'Pred: {pred_obj_name}'
f'Score: {score:.3f},\t\t',
f'IoU: {bbox_iou:.3f},',
)
print("AP @0.5-0.95: {:.5f}".format(AP))
# visualize bbox.
pred_bbox_img = arl_affpose_dataset_utils.draw_bbox_on_img(
image=image, scores=scores, obj_ids=obj_ids, boxes=obj_boxes)
# visualize affordance masks.
pred_aff_mask = arl_affpose_dataset_utils.get_segmentation_masks(
image=image,
obj_ids=aff_ids,
binary_masks=aff_binary_masks,
)
color_aff_mask = arl_affpose_dataset_utils.colorize_aff_mask(
pred_aff_mask)
color_aff_mask = cv2.addWeighted(pred_bbox_img, 0.5, color_aff_mask,
0.5, 0)
# get obj part mask.
pred_obj_part_mask = arl_affpose_dataset_utils.get_obj_part_mask(
image=image,
obj_part_ids=obj_part_ids,
aff_binary_masks=aff_binary_masks,
)
# visualize object masks.
pred_obj_mask = arl_affpose_dataset_utils.convert_obj_part_mask_to_obj_mask(
pred_obj_part_mask)
color_obj_mask = arl_affpose_dataset_utils.colorize_obj_mask(
pred_obj_mask)
color_obj_mask = cv2.addWeighted(pred_bbox_img, 0.5, color_obj_mask,
0.5, 0)
if SAVE_AND_EVAL_PRED:
# saving predictions.
_image_idx = target["image_id"].detach().numpy()[0]
_image_idx = str(1000000 + _image_idx)[1:]
gt_name = config.ARL_AFF_EVAL_SAVE_FOLDER + _image_idx + config.TEST_GT_EXT
pred_name = config.ARL_AFF_EVAL_SAVE_FOLDER + _image_idx + config.TEST_PRED_EXT
obj_part_name = config.ARL_AFF_EVAL_SAVE_FOLDER + _image_idx + config.TEST_OBJ_PART_EXT
cv2.imwrite(gt_name, target['aff_mask'])
cv2.imwrite(pred_name, pred_aff_mask)
cv2.imwrite(obj_part_name, pred_obj_part_mask)
# show plot.
if SHOW_IMAGES:
cv2.imshow('pred_bbox',
cv2.cvtColor(pred_bbox_img, cv2.COLOR_BGR2RGB))
cv2.imshow('pred_aff_mask',
cv2.cvtColor(color_aff_mask, cv2.COLOR_BGR2RGB))
cv2.imshow('pred_obj_part_mask',
cv2.cvtColor(color_obj_mask, cv2.COLOR_BGR2RGB))
cv2.waitKey(0)
# Confusion Matrix.
cm = sklearn_confusion_matrix(y_true=gt_obj_ids_list,
y_pred=pred_obj_ids_list)
print(f'\n{cm}')
# Plot Confusion Matrix.
# eval_utils.plot_confusion_matrix(cm, arl_affpose_dataset_utils.OBJ_NAMES)
# mAP
print("\nmAP @0.5-0.95: over {} test images is {:.3f}".format(
len(APs), np.mean(APs)))
if SAVE_AND_EVAL_PRED:
print()
# getting FwB.
os.chdir(config.MATLAB_SCRIPTS_DIR)
import matlab.engine
eng = matlab.engine.start_matlab()
Fwb = eng.evaluate_arl_affpose_affnet(config.ARL_AFF_EVAL_SAVE_FOLDER,
nargout=1)
os.chdir(config.ROOT_DIR_PATH)
def test_occlusion(self):
# load object ply files.
self.cld, self.cld_obj_centered, self.cld_obj_part_centered, \
self.obj_classes, self.obj_part_classes = load_arl_affpose_obj_ply_files.load_obj_ply_files()
obj_occlusion = np.zeros(shape=(len(self.data_loader),
len(self.obj_classes)))
obj_part_occlusion = np.zeros(shape=(len(self.data_loader),
len(self.obj_part_classes)))
# loop over dataset.
for i, (image, target) in enumerate(self.data_loader):
if i % 10 == 0:
print(f'{i}/{len(self.data_loader)} ..')
# format rgb.
image = np.squeeze(np.array(image))
image, target = arl_affpose_dataset_utils.format_target_data(
image, target)
H, W, C = image.shape[0], image.shape[1], image.shape[2]
# get image idx.
image_id = target['image_id'].item()
image_id = str(1000000 + image_id)[1:]
# load meta data.
meta_addr = self.data_loader.dataset.dataset.dataset_dir + 'meta/' + image_id + '_meta.mat'
meta = scio.loadmat(meta_addr)
# overlay obj mask on rgb images.
obj_mask = target['obj_mask']
obj_part_mask = target['obj_part_mask']
colour_label = arl_affpose_dataset_utils.colorize_obj_mask(
obj_mask)
colour_label = cv2.addWeighted(image, 0.35, colour_label, 0.65, 0)
# Img to draw 6-DoF Pose.
cv2_pose_img = colour_label.copy()
#######################################
#######################################
cam_cx = meta['cam_cx'][0][0]
cam_cy = meta['cam_cy'][0][0]
cam_fx = meta['cam_fx'][0][0]
cam_fy = meta['cam_fy'][0][0]
cam_mat = np.array([[cam_fx, 0, cam_cx], [0, cam_fy, cam_cy],
[0, 0, 1]])
cam_dist = np.array([0.0, 0.0, 0.0, 0.0, 0.0])
#######################################
# OBJECT
#######################################
obj_ids = np.array(meta['object_class_ids']).flatten()
for idx, obj_id in enumerate(obj_ids):
obj_id = int(obj_id)
obj_color = arl_affpose_dataset_utils.obj_color_map(obj_id)
# print(f"\tObject: {obj_id}, {self.obj_classes[int(obj_id) - 1]}")
obj_meta_idx = str(1000 + obj_id)[1:]
obj_r = meta['obj_rotation_' + str(obj_meta_idx)]
obj_t = meta['obj_translation_' + str(obj_meta_idx)]
obj_r = np.array(obj_r, dtype=np.float64).reshape(3, 3)
obj_t = np.array(obj_t, dtype=np.float64).reshape(-1, 3)
cv2_obj_label = np.zeros(shape=(obj_mask.shape),
dtype=np.uint8)
#######################################
# ITERATE OVER OBJ PARTS
#######################################
obj_part_ids = arl_affpose_dataset_utils.map_obj_id_to_obj_part_ids(
obj_id)
# print(f'\tobj_part_ids:{obj_part_ids}')
for obj_part_id in obj_part_ids:
obj_part_id = int(obj_part_id)
aff_id = arl_affpose_dataset_utils.map_obj_part_id_to_aff_id(
obj_part_id)
aff_color = arl_affpose_dataset_utils.aff_color_map(aff_id)
# print(f"\t\tAff: {aff_id}, {self.obj_part_classes[int(obj_part_id) - 1]}")
#######################################
# OBJ POSE
#######################################
# projecting 3D model to 2D image
obj_centered = self.cld_obj_centered[obj_part_id]
imgpts, jac = cv2.projectPoints(obj_centered * 1e3, obj_r,
obj_t * 1e3, cam_mat,
cam_dist)
cv2_pose_img = cv2.polylines(
cv2_pose_img, np.int32([np.squeeze(imgpts)]), True,
obj_color)
#######################################
# OBJ MASK
#######################################
# Draw obj mask.
cv2_drawpoints_img = np.zeros(shape=(obj_mask.shape),
dtype=np.uint8)
cv2_drawpoints_img = cv2.polylines(
cv2_drawpoints_img, np.int32([np.squeeze(imgpts)]),
False, (obj_id))
cv2_masked = np.ma.getmaskarray(
np.ma.masked_equal(cv2_drawpoints_img, obj_id))
# get contours.
# cv2_masked = np.ma.getmaskarray(np.ma.masked_equal(obj_mask, obj_id)) * cv2_masked
res = cv2.findContours(cv2_masked.astype(np.uint8),
cv2.RETR_LIST,
cv2.CHAIN_APPROX_NONE)
contours = res[-2] # for cv2 v3 and v4+ compatibility
# get obj mask.
cv2_obj_label = cv2.drawContours(cv2_obj_label,
contours,
contourIdx=-1,
color=(obj_id),
thickness=-1)
#######################################
# OBJ PART POSE
#######################################
obj_part_meta_idx = str(1000 + obj_part_id)[1:]
obj_part_r = meta['obj_part_rotation_' +
str(obj_part_meta_idx)]
obj_part_t = meta['obj_part_translation_' +
str(obj_part_meta_idx)]
# projecting 3D model to 2D image
obj_part_centered = self.cld_obj_part_centered[obj_part_id]
imgpts, jac = cv2.projectPoints(obj_part_centered * 1e3,
obj_part_r,
obj_part_t * 1e3, cam_mat,
cam_dist)
#######################################
# OBJ PART MASK
#######################################
# Draw obj mask.
cv2_obj_part_label = np.zeros(shape=(obj_part_mask.shape),
dtype=np.uint8)
cv2_drawpoints_img = np.zeros(shape=(obj_mask.shape),
dtype=np.uint8)
cv2_drawpoints_img = cv2.polylines(
cv2_drawpoints_img, np.int32([np.squeeze(imgpts)]),
False, (obj_part_id))
cv2_masked = np.ma.getmaskarray(
np.ma.masked_equal(cv2_drawpoints_img, obj_part_id))
# get contours.
res = cv2.findContours(cv2_masked.astype(np.uint8),
cv2.RETR_LIST,
cv2.CHAIN_APPROX_NONE)
contours = res[-2] # for cv2 v3 and v4+ compatibility
# get obj mask.
cv2_obj_part_label = cv2.drawContours(cv2_obj_part_label,
contours,
contourIdx=-1,
color=(obj_part_id),
thickness=-1)
#######################################
# OBJ PART Occlusion
#######################################
# actual.
masked_obj_part_label = np.ma.getmaskarray(
np.ma.masked_equal(obj_part_mask,
obj_part_id)).astype(np.uint8)
actual_points = np.count_nonzero(masked_obj_part_label)
# expected.
masked_cv2_obj_part_label = np.ma.getmaskarray(
np.ma.masked_equal(cv2_obj_part_label,
obj_part_id)).astype(np.uint8)
expected_points = np.count_nonzero(
masked_cv2_obj_part_label)
# calc occlusion.
percent_occlusion = actual_points / expected_points
obj_part_occlusion[i, obj_part_id - 1] = percent_occlusion
# print(f"\t\tObject Part: Id:{obj_part_id}, Name:{self.obj_part_classes[int(obj_part_id) - 1]}, "
# f"% Occlusion: {percent_occlusion * 100:.2f}")
#######################################
# OBJ Occlusion
#######################################
# actual.
masked_obj_label = np.ma.getmaskarray(
np.ma.masked_equal(obj_mask, obj_id)).astype(np.uint8)
actual_points = np.count_nonzero(masked_obj_label)
# expected.
masked_cv2_obj_label = np.ma.getmaskarray(
np.ma.masked_equal(cv2_obj_label, obj_id)).astype(np.uint8)
expected_points = np.count_nonzero(masked_cv2_obj_label)
# calc occlusion.
percent_occlusion = actual_points / expected_points
obj_occlusion[i, obj_id - 1] = percent_occlusion
# print(f"\tObject: Id:{obj_id}, Name:{self.obj_classes[int(obj_id) - 1]}, "
# f"% Occlusion: {percent_occlusion*100:.2f}")
#######################################
# Plotting
#######################################
# debugging.
# cv2.imshow('cv2_pose_img', cv2_pose_img)
# cv2.waitKey(0)
# debugging.
# cv2.imshow('label_bbox', masked_obj_label * 100)
# cv2.imshow('masked_cv2', masked_cv2_obj_label * 100)
# cv2.waitKey(0)
# debugging.
# cv2.imshow('cv2_obj_part_label', cv2_obj_part_label * 100)
# cv2.waitKey(0)
print()
for obj_id in range(len(self.obj_classes)):
_obj_occlusion = obj_occlusion[:, obj_id]
# get nonzero values.
idxs = np.nonzero(_obj_occlusion)[0]
if len(idxs) != 0:
masked_obj_occlusion = _obj_occlusion[idxs]
# standardize to 1.
masked_obj_occlusion += (1 - np.nanmax(masked_obj_occlusion))
# get mean and std.
avg_obj_occlusion = np.mean(masked_obj_occlusion)
std_obj_occlusion = np.std(masked_obj_occlusion)
print(f"Occlusion: "
f"\tMean: {avg_obj_occlusion * 100:.2f}, "
f"\tStd Dev: {std_obj_occlusion * 100:.2f}, "
f"\tObject: Id:{obj_id+1}, "
f"\tName:{self.obj_classes[int(obj_id)]}, ")
print()
for obj_part_id in range(len(self.obj_part_classes)):
_obj_part_occlusion = obj_part_occlusion[:, obj_part_id]
# get nonzero values.
idxs = np.nonzero(_obj_part_occlusion)[0]
if len(idxs) != 0:
masked_obj_part_occlusion = _obj_part_occlusion[idxs]
# standardize to 1.
masked_obj_part_occlusion += (
1 - np.nanmax(masked_obj_part_occlusion))
# get mean and std.
avg_obj_part_occlusion = np.mean(masked_obj_part_occlusion)
std_obj_part_occlusion = np.std(masked_obj_part_occlusion)
print(f"Occlusion: "
f"\tMean: {avg_obj_part_occlusion * 100:.2f}, "
f"\tStd Dev: {std_obj_part_occlusion * 100:.2f}, "
f"\tObject Part: Id:{obj_part_id+1}, "
f"\tName:{self.obj_part_classes[int(obj_part_id)]},")
