def get_rel_score(args, t, loader, model):
float_dtype = torch.FloatTensor
long_dtype = torch.LongTensor
num_samples = 0
all_losses = defaultdict(list)
total_iou = 0
total_boxes = 0
rel_score = 0
with torch.no_grad():
o_start = o_end = 0
t_start = t_end = 0
last_o_idx = last_t_idx = 0
b = 0
total_boxes = 0
total_iou = 0
for batch in loader:
batch = [tensor.cuda() for tensor in batch]
#batch = [tensor for tensor in batch]
masks = None
if len(batch) == 6:
imgs, objs, boxes, triples, obj_to_img, triple_to_img = batch
elif len(batch) == 8:
imgs, objs, boxes, masks, triples, obj_to_img, triple_to_img, triplet_masks = batch
#elif len(batch) == 7:
# imgs, objs, boxes, masks, triples, obj_to_img, triple_to_img = batch
predicates = triples[:, 1]
objs = objs.detach()
triples = triples.detach()
# Run the model as it has been run during training
model_masks = masks
model_out = model(objs,
triples,
obj_to_img,
boxes_gt=boxes,
masks_gt=model_masks)
# imgs_pred, boxes_pred, masks_pred, predicate_scores = model_out
imgs_pred, boxes_pred, masks_pred, objs_vec, layout, layout_boxes, layout_masks, obj_to_img, sg_context_pred, sg_context_pred_d, predicate_scores, obj_embeddings, pred_embeddings, triple_boxes_pred, triple_boxes_gt, triplet_masks_pred = model_out
num_samples += imgs.size(0)
if num_samples >= args.num_val_samples:
break
rel_score += relation_score(boxes_pred, boxes, masks_pred, masks,
model.vocab)
b += 1
total_iou += jaccard(boxes_pred, boxes)
total_boxes += boxes_pred.size(0)
rel_score = rel_score / b
avg_iou = total_iou / total_boxes
return rel_score, avg_iou
def get_rel_score(args, t, loader, model):
float_dtype = torch.FloatTensor
long_dtype = torch.LongTensor
num_samples = 0
all_losses = defaultdict(list)
total_iou = 0
total_boxes = 0
###################
if not os.path.isdir(args.output_dir):
os.mkdir(args.output_dir)
print('Created %s' % args.output_dir)
img_dir = args.output_dir + '/img_dir'
if not os.path.isdir(img_dir):
os.mkdir(img_dir)
print('Created %s' % img_dir)
##################
rel_score = 0
with torch.no_grad():
o_start = o_end = 0
t_start = t_end = 0
last_o_idx = last_t_idx = 0
b = 0
total_boxes = 0
total_iou = 0
for batch in loader:
#batch = [tensor.cuda() for tensor in batch]
batch = [tensor for tensor in batch]
masks = None
if len(batch) == 6:
imgs, objs, boxes, triples, obj_to_img, triple_to_img = batch
elif len(batch) == 7:
imgs, objs, boxes, masks, triples, obj_to_img, triple_to_img = batch
predicates = triples[:, 1]
objs = objs.detach()
triples = triples.detach()
# Run the model as it has been run during training
model_masks = masks
model_out = model(objs,
triples,
obj_to_img,
boxes_gt=boxes,
masks_gt=model_masks)
# imgs_pred, boxes_pred, masks_pred, predicate_scores = model_out
imgs_pred, boxes_pred, masks_pred, objs_vec, layout, layout_boxes, layout_masks, obj_to_img, sg_context_pred, sg_context_pred_d, predicate_scores = model_out
num_samples += imgs.size(0)
if num_samples >= args.num_val_samples:
break
rel_score += relation_score(boxes_pred, boxes, masks_pred, masks,
model.vocab)
b += 1
total_iou += jaccard(boxes_pred, boxes)
total_boxes += boxes_pred.size(0)
print(b)
print(total_boxes)
rel_score = rel_score / b
avg_iou = total_iou / total_boxes
# print('rel score:' rel_score)
# print('avg iou:' avg_iou)
return rel_score, avg_iou
def get_rel_score(args, t, loader, model):
float_dtype = torch.FloatTensor
long_dtype = torch.LongTensor
num_samples = 0
all_losses = defaultdict(list)
total_iou = 0
total_boxes = 0
###################
if not os.path.isdir(args.output_dir):
os.mkdir(args.output_dir)
print('Created %s' % args.output_dir)
img_dir = args.output_dir + '/img_dir'
if not os.path.isdir(img_dir):
os.mkdir(img_dir)
print('Created %s' % img_dir)
##################
rel_score = 0
with torch.no_grad():
o_start = o_end = 0
t_start = t_end = 0
last_o_idx = last_t_idx = 0
b = 0
total_boxes = 0
total_iou = 0
for batch in loader:
batch = [tensor.cuda() for tensor in batch]
#batch = [tensor for tensor in batch]
masks = None
if len(batch) == 6:
imgs, objs, boxes, triples, obj_to_img, triple_to_img = batch
elif len(batch) == 10:
imgs, objs, boxes, masks, triples, obj_to_img, triple_to_img, triplet_masks, triplet_contours, contours = batch
#elif len(batch) == 8:
# imgs, objs, boxes, masks, triples, obj_to_img, triple_to_img, triplet_masks = batch
#elif len(batch) == 7:
# imgs, objs, boxes, masks, triples, obj_to_img, triple_to_img = batch
predicates = triples[:, 1]
objs = objs.detach()
triples = triples.detach()
# Run the model as it has been run during training
model_masks = masks
model_out = model(objs,
triples,
obj_to_img,
boxes_gt=boxes,
masks_gt=model_masks)
# imgs_pred, boxes_pred, masks_pred, predicate_scores = model_out
#imgs_pred, boxes_pred, masks_pred, objs_vec, layout, layout_boxes, layout_masks, obj_to_img, sg_context_pred, sg_context_pred_d, predicate_scores, obj_embeddings, pred_embeddings, triple_boxes_pred, triple_boxes_gt = model_out
#imgs_pred, boxes_pred, masks_pred, objs_vec, layout, layout_boxes, layout_masks, obj_to_img, sg_context_pred, sg_context_pred_d, predicate_scores, obj_embeddings, pred_embeddings, triple_boxes_pred, triple_boxes_gt, triplet_masks_pred = model_out
imgs_pred, boxes_pred, masks_pred, objs_vec, layout, layout_boxes, layout_masks, obj_to_img, sg_context_pred, sg_context_pred_d, predicate_scores, obj_embeddings, pred_embeddings, triple_boxes_pred, triple_boxes_gt, triplet_masks_pred, triplet_contour_pred, contours_pred = model_out
num_samples += imgs.size(0)
if num_samples >= args.num_val_samples:
break
# if obj contours are predicted, derive bounding box from these;
# if GT boxes are passed in, layout_masks are GT boxes
if contours_pred is not None and boxes_pred is None:
boxes_gt = boxes
# get min-max bboxes, contour mean (not bbox mean), contour std dev (e.g. use predicted bboxes to estimate centers)
boxes_pred, mean_pts_pred, std_pts_pred = min_max_bbox_fr_contours(
contours_pred) # predicted
#boxes, mean_pts, std_pts = min_max_bbox_fr_contours(contours) # GT
# use geom center contour of contour to calculate relations score
# (can't calculate geometric center of contour in sg2im.data.utils.determine_box_relation()
#boxes_pred_mn = torch.cat([mean_pts_pred, mean_pts_pred], dim=1)
#boxes_mn = torch.cat([mean_pts, mean_pts], dim=1)
#pdb.set_trace()
if 0:
# visualize first contour of each batch
import matplotlib.pyplot as plt
cc = contours.clone().view(-1, 12, 2)
cp = contours_pred.view(-1, 12, 2).clone().detach()
cp = cp.cpu().numpy()
#bb = new_boxes[0].view(2,2)
bb = boxes[0].view(2, 2)
bbp = boxes_pred.clone().detach()
#bbp = new_boxes_pred.clone().detach()
bbp = bbp[0].view(2, 2)
fig, ax = plt.subplots()
#ax.imshow(masks[0])
# without mask, origin will be LLHC
ax.scatter(cc[0, :, 0], cc[0, :, 1], linewidth=0.5)
ax.scatter(cp[0, :, 0], cp[0, :, 1], linewidth=0.5)
ax.scatter(bb[:, 0], bb[:, 1], linewidth=1.0, marker="x")
ax.scatter(bbp[:, 0], bbp[:, 1], linewidth=1.0, marker="x")
plt.show()
#pdb.set_trace()
# use geometric center of contour l/r/t/b spatial relationships; use min-max box for surrounding/in
#rel_score += relation_score_cont(boxes_pred_mn, boxes_mn, masks_pred, masks, boxes_pred, boxes, model.vocab) # no mask, faske boxes
rel_score += relation_score(boxes_pred, boxes, masks_pred, masks,
model.vocab)
b += 1
#total_iou += jaccard(new_boxes_pred, boxes)
total_iou += jaccard(boxes_pred, boxes)
total_boxes += boxes_pred.size(0)
rel_score = rel_score / b
avg_iou = total_iou / total_boxes
# print('rel score:' rel_score)
# print('avg iou:' avg_iou)
return rel_score, avg_iou