Exemplo n.º 1
0
    def __getitem__(self, index):
        metainfo = GView(self.get_metainfo(index))
        feed_dict = GView()

        # scene annotations
        if self.incl_scene:
            feed_dict.scene = metainfo.scene
            feed_dict.update(gdef.annotate_objects(metainfo.scene))
            feed_dict.objects_raw = feed_dict.objects.copy()
            feed_dict.update(gdef.annotate_scene(metainfo.scene))

        # image
        feed_dict.image_index = metainfo.image_index
        feed_dict.image_filename = metainfo.image_filename
        if self.image_root is not None:
            feed_dict.image = Image.open(osp.join(self.image_root, feed_dict.image_filename)).convert('RGB')
            feed_dict.image, feed_dict.objects = self.image_transform(feed_dict.image, feed_dict.objects)

        # program
        feed_dict.program_raw = metainfo.program_raw
        feed_dict.program_seq = metainfo.program_seq
        feed_dict.program_tree = metainfo.program_tree
        feed_dict.program_qsseq = metainfo.program_qsseq
        feed_dict.program_qstree = metainfo.program_qstree
        feed_dict.question_type = metainfo.question_type

        # question
        feed_dict.answer = True

        return feed_dict.raw()
Exemplo n.º 2
0
    def __getitem__(self, index):
        metainfo = GView(self.get_metainfo(index))
        feed_dict = GView()

        # metainfo annotations
        if self.incl_scene:
            feed_dict.scene = metainfo.scene
            feed_dict.update(gdef.annotate_objects(metainfo.scene))
            if "objects" in feed_dict:
                # NB(Jiayuan Mao): in some datasets, object information might be completely unavailable.
                feed_dict.objects_raw = feed_dict.objects.copy()
            feed_dict.update(gdef.annotate_scene(metainfo.scene))

        # image
        feed_dict.image_index = metainfo.image_index
        feed_dict.image_filename = metainfo.image_filename
        if self.image_root is not None and feed_dict.image_filename is not None:
            feed_dict.image = Image.open(
                osp.join(self.image_root, feed_dict.image_filename)
            ).convert("RGB")
            feed_dict.image, feed_dict.objects = self.image_transform(
                feed_dict.image, feed_dict.objects
            )
        if self.depth_root is not None and feed_dict.image_filename is not None:
            depth_filename = feed_dict.image_filename.split(".")[0] + ".exr"
            feed_dict.depth = torch.tensor(
                load_depth(osp.join(self.depth_root, depth_filename))
            )

        # program
        if "program_raw" in metainfo:
            feed_dict.program_raw = metainfo.program_raw
            feed_dict.program_seq = metainfo.program_seq
            feed_dict.program_tree = metainfo.program_tree
            feed_dict.program_qsseq = metainfo.program_qsseq
            feed_dict.program_qstree = metainfo.program_qstree
        feed_dict.question_type = metainfo.question_type

        # question
        feed_dict.question_index = metainfo.question_index
        feed_dict.question_raw = metainfo.question
        feed_dict.question_raw_tokenized = metainfo.question_tokenized
        feed_dict.question_metainfo = gdef.annotate_question_metainfo(metainfo)
        feed_dict.question = metainfo.question_tokenized
        feed_dict.answer = gdef.canonize_answer(metainfo.answer, metainfo.question_type)
        feed_dict.update(gdef.annotate_question(metainfo))

        if self.question_transform is not None:
            self.question_transform(feed_dict)
        feed_dict.question = np.array(
            self.vocab.map_sequence(feed_dict.question), dtype="int64"
        )

        return feed_dict.raw()
Exemplo n.º 3
0
    def __getitem__(self, index):
        metainfo = GView(self.get_metainfo(index))
        feed_dict = GView()

        # scene annotations
        if self.incl_scene:
            feed_dict.scene = metainfo.scene
            feed_dict.update(gdef.annotate_objects(metainfo.scene))
            feed_dict.objects_raw = feed_dict.objects.copy()
            feed_dict.update(gdef.annotate_scene(metainfo.scene))

        # image
        feed_dict.image_index = metainfo.image_index
        feed_dict.image_filename = metainfo.image_filename

        # video
        feed_dict.video_folder = metainfo.video_folder
        video = []
        original_objects = feed_dict.objects
        if self.image_root is not None:
            feed_dict.image = Image.open(
                osp.join(self.image_root,
                         feed_dict.image_filename)).convert("RGB")
            feed_dict.image, feed_dict.objects = self.image_transform(
                feed_dict.image, feed_dict.objects)

        if self.image_root is not None and feed_dict.video_folder is not None:
            import glob

            for name in glob.glob(
                    osp.join(self.image_root, feed_dict.video_folder) +
                    "/*.png"):
                image = Image.open(name).convert("RGB")
                image, _ = self.image_transform(image, original_objects)
                video += [image]

            feed_dict.video = torch.cat(video)

        # program
        feed_dict.program_raw = metainfo.program_raw
        feed_dict.program_seq = metainfo.program_seq
        feed_dict.program_tree = metainfo.program_tree
        feed_dict.program_qsseq = metainfo.program_qsseq
        feed_dict.program_qstree = metainfo.program_qstree
        feed_dict.question_type = metainfo.question_type

        # question
        feed_dict.answer = True

        return feed_dict.raw()
Exemplo n.º 4
0
    def __getitem__(self, index):
        metainfo = GView(self.get_metainfo(index))
        feed_dict = GView()

        # metainfo annotations
        if self.incl_scene:
            feed_dict.scene = metainfo.scene
            feed_dict.update(gdef.annotate_objects(metainfo.scene))
            if "objects" in feed_dict:
                # NB(Jiayuan Mao): in some datasets, object information might be completely unavailable.
                feed_dict.objects_raw = feed_dict.objects.copy()
            feed_dict.update(gdef.annotate_scene(metainfo.scene))

        # image
        feed_dict.image_index = metainfo.image_index
        feed_dict.image_filename = metainfo.image_filename
        # video
        feed_dict.video_folder = metainfo.video_folder
        video = []
        original_objects = feed_dict.objects
        if self.image_root is not None and feed_dict.image_filename is not None:
            feed_dict.image = Image.open(
                osp.join(self.image_root,
                         feed_dict.image_filename)).convert("RGB")
            feed_dict.image, feed_dict.objects = self.image_transform(
                feed_dict.image, feed_dict.objects)

            # print("Image:", feed_dict.image.shape)
            # print(feed_dict.objects)

        if self.image_root is not None and feed_dict.video_folder is not None:
            import glob

            for name in glob.glob(
                    osp.join(self.image_root, feed_dict.video_folder) +
                    "/*.png"):
                image = Image.open(name).convert("RGB")
                image, _ = self.image_transform(image, original_objects)
                video += [image]

            feed_dict.video = torch.stack(video)

            # Tensor
            # print("Video:", feed_dict.video.shape)

        # program
        if "program_raw" in metainfo:
            feed_dict.program_raw = metainfo.program_raw
            feed_dict.program_seq = metainfo.program_seq
            feed_dict.program_tree = metainfo.program_tree
            feed_dict.program_qsseq = metainfo.program_qsseq
            feed_dict.program_qstree = metainfo.program_qstree
        feed_dict.question_type = metainfo.question_type

        # question
        feed_dict.question_index = metainfo.question_index
        feed_dict.question_raw = metainfo.question
        feed_dict.question_raw_tokenized = metainfo.question_tokenized
        feed_dict.question_metainfo = gdef.annotate_question_metainfo(metainfo)
        feed_dict.question = metainfo.question_tokenized
        feed_dict.answer = gdef.canonize_answer(metainfo.answer,
                                                metainfo.question_type)
        feed_dict.update(gdef.annotate_question(metainfo))

        if self.question_transform is not None:
            self.question_transform(feed_dict)
        feed_dict.question = np.array(self.vocab.map_sequence(
            feed_dict.question),
                                      dtype="int64")

        return feed_dict.raw()