def forward(self, gt_imgs, objs, triples, obj_to_img, boxes_gt=None, masks_gt=None, attributes=None,
gt_train=False, test_mode=False, use_gt_box=False, features=None):
O, T = objs.size(0), triples.size(0)
obj_vecs, pred_vecs = self.scene_graph_to_vectors(objs, triples, attributes)
box_vecs, mask_vecs, scene_layout_vecs, wrong_layout_vecs = \
self.create_components_vecs(gt_imgs, boxes_gt, obj_to_img, objs, obj_vecs, gt_train, features)
# Generate Boxes
boxes_pred = self.box_net(box_vecs)
# Generate Masks
mask_scores = self.mask_net(mask_vecs.view(O, -1, 1, 1))
masks_pred = mask_scores.squeeze(1).sigmoid()
H, W = self.image_size
if test_mode:
boxes = boxes_gt if use_gt_box else boxes_pred
masks = masks_gt if masks_gt is not None else masks_pred
gt_layout = None
pred_layout = masks_to_layout(scene_layout_vecs, boxes, masks, obj_to_img, H, W, test_mode=True)
wrong_layout = None
imgs_pred = self.pix2pix(pred_layout)
else:
gt_layout = masks_to_layout(scene_layout_vecs, boxes_gt, masks_gt, obj_to_img, H, W, test_mode=False)
pred_layout = masks_to_layout(scene_layout_vecs, boxes_gt, masks_pred, obj_to_img, H, W, test_mode=False)
wrong_layout = masks_to_layout(wrong_layout_vecs, boxes_gt, masks_gt, obj_to_img, H, W, test_mode=False)
imgs_pred = self.pix2pix(gt_layout)
return imgs_pred, boxes_pred, masks_pred, gt_layout, pred_layout, wrong_layout
def forward(self,
gt_imgs,
objs,
triples,
obj_to_img,
boxes_gt=None,
masks_gt=None,
attributes=None,
gt_train=False,
test_mode=False,
use_gt_box=False,
features=None,
drop_box_idx=None,
drop_feat_idx=None,
src_image=None):
O, T = objs.size(0), triples.size(0)
obj_vecs, pred_vecs = self.scene_graph_to_vectors(
objs, triples, attributes)
box_vecs, mask_vecs, scene_layout_vecs, wrong_layout_vecs = \
self.create_components_vecs(gt_imgs, boxes_gt, obj_to_img, objs, obj_vecs, features
,drop_box_idx=drop_box_idx, drop_feat_idx=drop_feat_idx, src_image=src_image)
# Generate Boxes
boxes_pred = self.box_net(box_vecs)
vg = True
# Generate Masks
# Mask prediction network
masks_pred = None
if self.mask_net is not None:
mask_scores = self.mask_net(mask_vecs.view(O, -1, 1, 1))
masks_pred = mask_scores.squeeze(1).sigmoid()
H, W = self.image_size
if vg:
layout_boxes = boxes_pred if boxes_gt is None else boxes_gt
#layout_masks = masks_pred if masks_gt is None else masks_gt
masks_gt = masks_pred if masks_gt is None else masks_gt
#masks_pred = layout_boxes
if test_mode:
boxes = boxes_gt if use_gt_box else boxes_pred
masks = masks_gt if masks_gt is not None else masks_pred
gt_layout = None
pred_layout = masks_to_layout(scene_layout_vecs,
boxes,
masks,
obj_to_img,
H,
W,
test_mode=True)
wrong_layout = None
imgs_pred = self.layout_to_image(pred_layout)
else:
gt_layout = masks_to_layout(scene_layout_vecs,
boxes_gt,
masks_gt,
obj_to_img,
H,
W,
test_mode=False)
pred_layout = masks_to_layout(scene_layout_vecs,
boxes_gt,
masks_pred,
obj_to_img,
H,
W,
test_mode=False)
wrong_layout = masks_to_layout(wrong_layout_vecs,
boxes_gt,
masks_gt,
obj_to_img,
H,
W,
test_mode=False)
imgs_pred = self.layout_to_image(gt_layout)
return imgs_pred, boxes_pred, masks_pred, gt_layout, pred_layout, wrong_layout
def forward(self,
gt_imgs,
objs,
gt_fmaps,
obj_to_img,
boxes_gt=None,
masks_gt=None,
attributes=None,
test_mode=False,
use_gt_box=False,
features=None):
# O, T = objs.size(0), triples.size(0)
# obj_vecs, pred_vecs = self.scene_graph_to_vectors(objs, triples, attributes)
assert isinstance(
self.gconv, nn.Linear
), "self.gconv is not Linear, but currently we do not input scene graph"
obj_vecs = self.gconv(gt_fmaps)
box_vecs, mask_vecs, scene_layout_vecs, wrong_layout_vecs, obj_repr, crops = \
self.create_components_vecs(gt_imgs, boxes_gt, obj_to_img, objs, obj_vecs, features)
# Generate Boxes
boxes_pred = self.box_net(
box_vecs) if self.box_net is not None else None
# Generate Masks
mask_scores = self.mask_net(mask_vecs.view(objs.size(0), -1, 1, 1))
masks_pred = mask_scores.squeeze(1).sigmoid()
H, W = self.image_size
if test_mode:
boxes = boxes_gt if use_gt_box else boxes_pred
masks = masks_gt if masks_gt is not None else masks_pred
gt_layout = None
pred_layout = masks_to_layout(scene_layout_vecs,
boxes,
masks,
obj_to_img,
H,
W,
test_mode=True)
wrong_layout = None
imgs_pred = self.layout_to_image(pred_layout)
else:
# gt_layout = masks_to_layout(scene_layout_vecs, boxes_gt, masks_gt, obj_to_img, H, W, test_mode=False)
gt_layout = None
pred_layout = masks_to_layout(scene_layout_vecs,
boxes_gt,
masks_pred,
obj_to_img,
H,
W,
test_mode=False)
# wrong_layout = masks_to_layout(wrong_layout_vecs, boxes_gt, masks_gt, obj_to_img, H, W, test_mode=False)
wrong_layout = masks_to_layout(wrong_layout_vecs,
boxes_gt,
masks_pred,
obj_to_img,
H,
W,
test_mode=False)
# imgs_pred = self.layout_to_image(gt_layout)
imgs_pred = self.layout_to_image(pred_layout)
return imgs_pred, boxes_pred, masks_pred, gt_layout, pred_layout, wrong_layout, obj_repr, crops