def __getitem__(self, index):
metainfo = GView(self.get_metainfo(index))
feed_dict = GView()
# metainfo annotations
if self.incl_scene:
feed_dict.scene = metainfo.scene
feed_dict.update(gdef.annotate_objects(metainfo.scene))
if "objects" in feed_dict:
# NB(Jiayuan Mao): in some datasets, object information might be completely unavailable.
feed_dict.objects_raw = feed_dict.objects.copy()
feed_dict.update(gdef.annotate_scene(metainfo.scene))
# image
feed_dict.image_index = metainfo.image_index
feed_dict.image_filename = metainfo.image_filename
if self.image_root is not None and feed_dict.image_filename is not None:
feed_dict.image = Image.open(
osp.join(self.image_root, feed_dict.image_filename)
).convert("RGB")
feed_dict.image, feed_dict.objects = self.image_transform(
feed_dict.image, feed_dict.objects
)
if self.depth_root is not None and feed_dict.image_filename is not None:
depth_filename = feed_dict.image_filename.split(".")[0] + ".exr"
feed_dict.depth = torch.tensor(
load_depth(osp.join(self.depth_root, depth_filename))
)
# program
if "program_raw" in metainfo:
feed_dict.program_raw = metainfo.program_raw
feed_dict.program_seq = metainfo.program_seq
feed_dict.program_tree = metainfo.program_tree
feed_dict.program_qsseq = metainfo.program_qsseq
feed_dict.program_qstree = metainfo.program_qstree
feed_dict.question_type = metainfo.question_type
# question
feed_dict.question_index = metainfo.question_index
feed_dict.question_raw = metainfo.question
feed_dict.question_raw_tokenized = metainfo.question_tokenized
feed_dict.question_metainfo = gdef.annotate_question_metainfo(metainfo)
feed_dict.question = metainfo.question_tokenized
feed_dict.answer = gdef.canonize_answer(metainfo.answer, metainfo.question_type)
feed_dict.update(gdef.annotate_question(metainfo))
if self.question_transform is not None:
self.question_transform(feed_dict)
feed_dict.question = np.array(
self.vocab.map_sequence(feed_dict.question), dtype="int64"
)
return feed_dict.raw()
def __getitem__(self, index):
metainfo = GView(self.get_metainfo(index))
feed_dict = GView()
# metainfo annotations
if self.incl_scene:
feed_dict.scene = metainfo.scene
feed_dict.update(gdef.annotate_objects(metainfo.scene))
if "objects" in feed_dict:
# NB(Jiayuan Mao): in some datasets, object information might be completely unavailable.
feed_dict.objects_raw = feed_dict.objects.copy()
feed_dict.update(gdef.annotate_scene(metainfo.scene))
# image
feed_dict.image_index = metainfo.image_index
feed_dict.image_filename = metainfo.image_filename
# video
feed_dict.video_folder = metainfo.video_folder
video = []
original_objects = feed_dict.objects
if self.image_root is not None and feed_dict.image_filename is not None:
feed_dict.image = Image.open(
osp.join(self.image_root,
feed_dict.image_filename)).convert("RGB")
feed_dict.image, feed_dict.objects = self.image_transform(
feed_dict.image, feed_dict.objects)
# print("Image:", feed_dict.image.shape)
# print(feed_dict.objects)
if self.image_root is not None and feed_dict.video_folder is not None:
import glob
for name in glob.glob(
osp.join(self.image_root, feed_dict.video_folder) +
"/*.png"):
image = Image.open(name).convert("RGB")
image, _ = self.image_transform(image, original_objects)
video += [image]
feed_dict.video = torch.stack(video)
# Tensor
# print("Video:", feed_dict.video.shape)
# program
if "program_raw" in metainfo:
feed_dict.program_raw = metainfo.program_raw
feed_dict.program_seq = metainfo.program_seq
feed_dict.program_tree = metainfo.program_tree
feed_dict.program_qsseq = metainfo.program_qsseq
feed_dict.program_qstree = metainfo.program_qstree
feed_dict.question_type = metainfo.question_type
# question
feed_dict.question_index = metainfo.question_index
feed_dict.question_raw = metainfo.question
feed_dict.question_raw_tokenized = metainfo.question_tokenized
feed_dict.question_metainfo = gdef.annotate_question_metainfo(metainfo)
feed_dict.question = metainfo.question_tokenized
feed_dict.answer = gdef.canonize_answer(metainfo.answer,
metainfo.question_type)
feed_dict.update(gdef.annotate_question(metainfo))
if self.question_transform is not None:
self.question_transform(feed_dict)
feed_dict.question = np.array(self.vocab.map_sequence(
feed_dict.question),
dtype="int64")
return feed_dict.raw()