Esempio n. 1
0
    def __init__(self, config, constants):
        self.none_action = config["num_actions"]
        self.landmark_names = get_all_landmark_names()

        self.text_module = TextPointerModule(
            emb_dim=constants["word_emb_dim"],
            hidden_dim=constants["lstm_emb_dim"],
            vocab_size=config["vocab_size"])
        self.final_module = SegmentationFinalModule(
            text_module=self.text_module,
            text_emb_size=4 * constants["lstm_emb_dim"])
        if torch.cuda.is_available():
            self.text_module.cuda()
            self.final_module.cuda()
Esempio n. 2
0
 def __init__(self, config, constants):
     AbstractModel.__init__(self, config, constants)
     self.none_action = config["num_actions"]
     self.image_module = ImagePositionResnetModule(
         image_emb_size=constants["image_emb_dim"],
         input_num_channels=3 * constants["max_num_images"],
         image_height=config["image_height"],
         image_width=config["image_width"])
     if config["use_pointer_model"]:
         self.text_module = TextPointerModule(
             emb_dim=constants["word_emb_dim"],
             hidden_dim=constants["lstm_emb_dim"],
             vocab_size=config["vocab_size"])
     else:
         self.text_module = TextSimpleModule(
             emb_dim=constants["word_emb_dim"],
             hidden_dim=constants["lstm_emb_dim"],
             vocab_size=config["vocab_size"])
     # total_emb_size = (constants["image_emb_dim"]
     #                   + constants["lstm_emb_dim"])
     total_emb_size = constants["image_emb_dim"]
     final_module = MultimodalSimplePositionModule(
         image_module=self.image_module,
         text_module=self.text_module,
         total_emb_size=total_emb_size,
         num_grid_x=8,
         num_grid_y=8,
         num_grid_pose=24)
     self.final_module = final_module
     if torch.cuda.is_available():
         self.image_module.cuda()
         self.text_module.cuda()
         self.final_module.cuda()
 def __init__(self, config, constants):
     AbstractModel.__init__(self, config, constants)
     self.none_action = config["num_actions"]
     self.image_module = ImageTextKernelResnetModule(
         image_emb_size=constants["image_emb_dim"],
         input_num_channels=3,
         image_height=config["image_height"],
         image_width=config["image_width"],
         text_emb_size=constants["lstm_emb_dim"],
         using_recurrence=True)
     self.image_recurrence_module = RecurrenceSimpleModule(
         input_emb_dim=constants["image_emb_dim"],
         output_emb_dim=constants["image_emb_dim"])
     if config["use_pointer_model"]:
         self.text_module = TextPointerModule(
             emb_dim=constants["word_emb_dim"],
             hidden_dim=constants["lstm_emb_dim"],
             vocab_size=config["vocab_size"])
     else:
         self.text_module = TextSimpleModule(
             emb_dim=constants["word_emb_dim"],
             hidden_dim=constants["lstm_emb_dim"],
             vocab_size=config["vocab_size"])
     self.action_module = ActionSimpleModule(
         num_actions=config["num_actions"],
         action_emb_size=constants["action_emb_dim"])
     if config["use_pointer_model"]:
         total_emb_size = (constants["image_emb_dim"] +
                           4 * constants["lstm_emb_dim"] +
                           constants["action_emb_dim"])
     else:
         total_emb_size = (constants["image_emb_dim"] +
                           constants["lstm_emb_dim"] +
                           constants["action_emb_dim"])
     final_module = MultimodalTextKernelRecurrentSimpleModule(
         image_module=self.image_module,
         image_recurrence_module=self.image_recurrence_module,
         text_module=self.text_module,
         action_module=self.action_module,
         total_emb_size=total_emb_size,
         num_actions=config["num_actions"])
     self.final_module = final_module
     if torch.cuda.is_available():
         self.image_module.cuda()
         self.image_recurrence_module.cuda()
         self.text_module.cuda()
         self.action_module.cuda()
         self.final_module.cuda()
 def __init__(self, config, constants):
     AbstractModel.__init__(self, config, constants)
     self.none_action = config["num_actions"]
     landmark_names = get_all_landmark_names()
     self.radius_module = RadiusModule(15)
     self.angle_module = AngleModule(48)
     self.landmark_module = LandmarkModule(63)
     self.image_module = SymbolicImageModule(
         landmark_names=landmark_names,
         radius_module=self.radius_module,
         angle_module=self.angle_module,
         landmark_module=self.landmark_module)
     if config["use_pointer_model"]:
         self.text_module = TextPointerModule(
             emb_dim=constants["word_emb_dim"],
             hidden_dim=constants["lstm_emb_dim"],
             vocab_size=config["vocab_size"])
     else:
         self.text_module = TextSimpleModule(
             emb_dim=constants["word_emb_dim"],
             hidden_dim=constants["lstm_emb_dim"],
             vocab_size=config["vocab_size"])
     self.action_module = ActionSimpleModule(
         num_actions=config["num_actions"],
         action_emb_size=constants["action_emb_dim"])
     total_emb_size = (32 * 3 * 63
                       + constants["lstm_emb_dim"]
                       + constants["action_emb_dim"])
     final_module = MultimodalSimpleModule(
         image_module=self.image_module,
         text_module=self.text_module,
         action_module=self.action_module,
         total_emb_size=total_emb_size,
         num_actions=config["num_actions"])
     self.final_module = final_module
     if torch.cuda.is_available():
         self.image_module.cuda()
         self.text_module.cuda()
         self.action_module.cuda()
         self.final_module.cuda()
         self.radius_module.cuda()
         self.angle_module.cuda()
         self.landmark_module.cuda()
Esempio n. 5
0
    def __init__(self, config, constants):
        AbstractIncrementalModel.__init__(self, config, constants)
        self.none_action = config["num_actions"]
        self.image_module = ImageResnetModule(
            image_emb_size=constants["image_emb_dim"],
            input_num_channels=3,
            image_height=config["image_height"],
            image_width=config["image_width"],
            using_recurrence=True)
        self.num_cameras = 1
        self.image_recurrence_module = IncrementalRecurrenceSimpleModule(
            input_emb_dim=(constants["image_emb_dim"] * self.num_cameras + constants["action_emb_dim"]),
            output_emb_dim=constants["image_emb_dim"])
        if config["use_pointer_model"]:
            self.text_module = TextPointerModule(
                emb_dim=constants["word_emb_dim"],
                hidden_dim=constants["lstm_emb_dim"],
                vocab_size=config["vocab_size"])
        else:
            self.text_module = TextBiLSTMModule(
                emb_dim=constants["word_emb_dim"],
                hidden_dim=constants["lstm_emb_dim"],
                vocab_size=config["vocab_size"])
        self.action_module = ActionSimpleModule(
            num_actions=config["num_actions"],
            action_emb_size=constants["action_emb_dim"])
        if config["use_pointer_model"]:
            total_emb_size = (constants["image_emb_dim"]
                              + 4 * constants["lstm_emb_dim"]
                              + constants["action_emb_dim"])
        else:
            total_emb_size = ((self.num_cameras + 1) * constants["image_emb_dim"]
                              + 2 * constants["lstm_emb_dim"]
                              + constants["action_emb_dim"])

        if config["do_action_prediction"]:
            self.action_prediction_module = ActionPredictionModule(
                2 * self.num_cameras * constants["image_emb_dim"], constants["image_emb_dim"], config["num_actions"])
        else:
            self.action_prediction_module = None

        if config["do_temporal_autoencoding"]:
            self.temporal_autoencoder_module = TemporalAutoencoderModule(
                self.action_module, self.num_cameras * constants["image_emb_dim"],
                constants["action_emb_dim"], constants["image_emb_dim"])
        else:
            self.temporal_autoencoder_module = None

        if config["do_object_detection"]:
            self.landmark_names = get_all_landmark_names()
            self.object_detection_module = ObjectDetectionModule(
                image_module=self.image_module, image_emb_size=self.num_cameras * constants["image_emb_dim"], num_objects=67)
        else:
            self.object_detection_module = None

        if config["do_symbolic_language_prediction"]:
            self.symbolic_language_prediction_module = SymbolicLanguagePredictionModule(
                total_emb_size=2 * constants["lstm_emb_dim"])
        else:
            self.symbolic_language_prediction_module = None

        if config["do_goal_prediction"]:
            self.goal_prediction_module = GoalPredictionModule(
                total_emb_size=32)
        else:
            self.goal_prediction_module = None

        final_module = TmpIncrementalMultimodalDenseValtsRecurrentSimpleModule(
            image_module=self.image_module,
            image_recurrence_module=self.image_recurrence_module,
            text_module=self.text_module,
            action_module=self.action_module,
            total_emb_size=total_emb_size,
            num_actions=config["num_actions"])
        self.final_module = final_module
        if torch.cuda.is_available():
            self.image_module.cuda()
            self.image_recurrence_module.cuda()
            self.text_module.cuda()
            self.action_module.cuda()
            self.final_module.cuda()
            if self.action_prediction_module is not None:
                self.action_prediction_module.cuda()
            if self.temporal_autoencoder_module is not None:
                self.temporal_autoencoder_module.cuda()
            if self.object_detection_module is not None:
                self.object_detection_module.cuda()
            if self.symbolic_language_prediction_module is not None:
                self.symbolic_language_prediction_module.cuda()
            if self.goal_prediction_module is not None:
                self.goal_prediction_module.cuda()
Esempio n. 6
0
class TextSegmentationModel(object):
    def __init__(self, config, constants):
        self.none_action = config["num_actions"]
        self.landmark_names = get_all_landmark_names()

        self.text_module = TextPointerModule(
            emb_dim=constants["word_emb_dim"],
            hidden_dim=constants["lstm_emb_dim"],
            vocab_size=config["vocab_size"])
        self.final_module = SegmentationFinalModule(
            text_module=self.text_module,
            text_emb_size=4 * constants["lstm_emb_dim"])
        if torch.cuda.is_available():
            self.text_module.cuda()
            self.final_module.cuda()

    def get_segmentation_probs(self, agent_observed_state_list):
        for aos in agent_observed_state_list:
            assert isinstance(aos, AgentObservedState)
        # print "batch size:", len(agent_observed_state_list)

        # sort list by instruction length
        agent_observed_state_list = sorted(
            agent_observed_state_list,
            key=lambda aos_: len(aos_.get_instruction()),
            reverse=True)

        instructions = [
            aos.get_instruction() for aos in agent_observed_state_list
        ]
        read_pointers = [
            aos.get_read_pointers() for aos in agent_observed_state_list
        ]
        instructions_batch = (instructions, read_pointers)
        probs_batch = self.final_module(instructions_batch)

        return probs_batch

    def load_saved_model(self, load_dir):
        if torch.cuda.is_available():
            torch_load = torch.load
        else:
            torch_load = lambda f_: torch.load(f_,
                                               map_location=lambda s_, l_: s_)
        text_module_path = os.path.join(load_dir, "text_module_state.bin")
        self.text_module.load_state_dict(torch_load(text_module_path))
        final_module_path = os.path.join(load_dir, "final_module_state.bin")
        self.final_module.load_state_dict(torch_load(final_module_path))

    def save_model(self, save_dir):
        if not os.path.exists(save_dir):
            os.makedirs(save_dir)

        # save state file for image nn
        text_module_path = os.path.join(save_dir, "text_module_state.bin")
        torch.save(self.text_module.state_dict(), text_module_path)
        # save state file for final nn
        final_module_path = os.path.join(save_dir, "final_module_state.bin")
        torch.save(self.final_module.state_dict(), final_module_path)

    def get_parameters(self):
        parameters = list(self.text_module.parameters())
        parameters += list(self.final_module.parameters())
        return parameters

    def get_named_parameters(self):
        named_parameters = list(self.text_module.named_parameters())
        named_parameters += list(self.final_module.named_parameters())
        return named_parameters