def __init__(self, config, constants):
        AbstractIncrementalModel.__init__(self, config, constants)
        self.none_action = config["num_actions"]
        self.image_module = MisraImageModule(
            image_emb_size=constants["image_emb_dim"],
            input_num_channels=3 * 5,
            image_height=config["image_height"],
            image_width=config["image_width"],
            using_recurrence=True)
        self.num_cameras = 1
        if config["use_pointer_model"]:
            raise NotImplementedError()
        else:
            self.text_module = ChaplotHouseTextModule(
                emb_dim=32,
                hidden_dim=256,
                vocab_size=config["vocab_size"],
                image_height=config["image_height"],
                image_width=config["image_width"])

        self.final_module = TmpHouseIncrementalMisraFinal(
            image_module=self.image_module, text_module=self.text_module)
        if torch.cuda.is_available():
            self.image_module.cuda()
            self.text_module.cuda()
            self.final_module.cuda()
Beispiel #2
0
 def __init__(self, config, constants):
     AbstractIncrementalModel.__init__(self, config, constants)
     self.none_action = config["num_actions"]
     self.image_module = ImageResnetModule(
         image_emb_size=constants["image_emb_dim"],
         input_num_channels=3,
         image_height=config["image_height"],
         image_width=config["image_width"],
         using_recurrence=True)
     self.image_recurrence_module = IncrementalRecurrenceSimpleModule(
         input_emb_dim=constants["image_emb_dim"],
         output_emb_dim=constants["image_emb_dim"])
     if config["use_pointer_model"]:
         raise AssertionError("Not implemented")
         # self.text_module = TextPointerModule(
         #     emb_dim=constants["word_emb_dim"],
         #     hidden_dim=constants["lstm_emb_dim"],
         #     vocab_size=config["vocab_size"])
     else:
         self.text_module = TextImplicitFactorizationModule(
             emb_dim=constants["word_emb_dim"],
             hidden_dim=constants["lstm_emb_dim"],
             vocab_size=config["vocab_size"],
             num_factors=2,
             factors_vocabulary_size=60,
             factors_embedding_size=250)
     self.action_module = ActionSimpleModule(
         num_actions=config["num_actions"],
         action_emb_size=constants["action_emb_dim"])
     if config["use_pointer_model"]:
         total_emb_size = (constants["image_emb_dim"] +
                           4 * constants["lstm_emb_dim"] +
                           constants["action_emb_dim"])
     else:
         total_emb_size = (constants["image_emb_dim"] + 2 * 250 +
                           constants["action_emb_dim"])
     final_module = IncrementalMultimodalRecurrentSimpleModule(
         image_module=self.image_module,
         image_recurrence_module=self.image_recurrence_module,
         text_module=self.text_module,
         action_module=self.action_module,
         total_emb_size=total_emb_size,
         num_actions=config["num_actions"])
     self.final_module = final_module
     if torch.cuda.is_available():
         self.image_module.cuda()
         self.image_recurrence_module.cuda()
         self.text_module.cuda()
         self.action_module.cuda()
         self.final_module.cuda()
    def __init__(self, config, constants):
        AbstractIncrementalModel.__init__(self, config, constants)
        self.none_action = config["num_actions"]

        self.config = config
        self.constants = constants

        # CNN over images - using SimpleImage for testing for now!
        self.image_module = ImageCnnEmnlp(
            image_emb_size=config["image_emb_dim"],
            input_num_channels=3 *
            5,  #3 channels per image - 5 images in history
            image_height=config["image_height"],
            image_width=config["image_width"])

        # this is somewhat counter intuitivie - emb_dim is the word size
        # hidden_size is the output size
        self.text_module = TextSimpleModule(emb_dim=config["word_emb_dim"],
                                            hidden_dim=config["lstm_emb_dim"],
                                            vocab_size=config["vocab_size"])

        self.previous_action_module = ActionSimpleModule(
            num_actions=config["no_actions"],
            action_emb_size=config["previous_action_embedding_dim"])

        self.previous_block_module = ActionSimpleModule(
            num_actions=config["no_blocks"],
            action_emb_size=config["previous_block_embedding_dim"])

        self.final_module = IncrementalMultimodalEmnlp(
            image_module=self.image_module,
            text_module=self.text_module,
            previous_action_module=self.previous_action_module,
            previous_block_module=self.previous_block_module,
            input_embedding_size=config["lstm_emb_dim"] +
            config["image_emb_dim"] + config["previous_action_embedding_dim"] +
            config["previous_block_embedding_dim"],
            output_hidden_size=config["h1_hidden_dim"],
            blocks_hidden_size=config["no_blocks"],
            directions_hidden_size=config["no_actions"],
            max_episode_length=(constants["horizon"] + 5))

        if torch.cuda.is_available():
            self.image_module.cuda()
            self.text_module.cuda()
            self.previous_action_module.cuda()
            self.previous_block_module.cuda()
            self.final_module.cuda()
 def __init__(self, config, constants):
     AbstractIncrementalModel.__init__(self, config, constants)
     self.none_action = config["num_actions"]
     self.image_module = ImageTextKernelResnetModule(
         image_emb_size=constants["image_emb_dim"],
         input_num_channels=3,
         image_height=config["image_height"],
         image_width=config["image_width"],
         text_emb_size=constants["lstm_emb_dim"],
         using_recurrence=True)
     self.image_recurrence_module = IncrementalRecurrenceSimpleModule(
         input_emb_dim=constants["image_emb_dim"],
         output_emb_dim=constants["image_emb_dim"])
     if config["use_pointer_model"]:
         self.text_module = TextPointerModule(
             emb_dim=constants["word_emb_dim"],
             hidden_dim=constants["lstm_emb_dim"],
             vocab_size=config["vocab_size"])
     else:
         self.text_module = TextSimpleModule(
             emb_dim=constants["word_emb_dim"],
             hidden_dim=constants["lstm_emb_dim"],
             vocab_size=config["vocab_size"])
     self.action_module = ActionSimpleModule(
         num_actions=config["num_actions"],
         action_emb_size=constants["action_emb_dim"])
     if config["use_pointer_model"]:
         total_emb_size = (constants["image_emb_dim"] +
                           4 * constants["lstm_emb_dim"] +
                           constants["action_emb_dim"])
     else:
         total_emb_size = (constants["image_emb_dim"] +
                           constants["lstm_emb_dim"] +
                           constants["action_emb_dim"])
     final_module = IncrementalMultimodalTextKernelRecurrentSimpleModule(
         image_module=self.image_module,
         image_recurrence_module=self.image_recurrence_module,
         text_module=self.text_module,
         action_module=self.action_module,
         total_emb_size=total_emb_size,
         num_actions=config["num_actions"])
     self.final_module = final_module
     if torch.cuda.is_available():
         self.image_module.cuda()
         self.image_recurrence_module.cuda()
         self.text_module.cuda()
         self.action_module.cuda()
         self.final_module.cuda()
Beispiel #5
0
    def __init__(self, config, constants):
        AbstractIncrementalModel.__init__(self, config, constants)
        self.none_action = config["num_actions"]
        self.image_module = UnetImageModule(
            image_emb_size=constants["image_emb_dim"],
            input_num_channels=3,
            image_height=config["image_height"],
            image_width=config["image_width"],
            using_recurrence=True)
        num_channels, image_height, image_width = 64, 32, 32
        self.num_cameras = 1

        if config["use_pointer_model"]:
            raise NotImplementedError()
        else:
            self.text_module = ChaplotTextModule(
                emb_dim=32,
                hidden_dim=256,
                vocab_size=config["vocab_size"],
                image_height=image_height,
                image_width=image_width)

        if config["do_object_detection"]:
            self.landmark_names = get_all_landmark_names()
            self.object_detection_module = PixelIdentificationModule(
                num_channels=num_channels, num_objects=67)
        else:
            self.object_detection_module = None

        if config["do_goal_prediction"]:
            self.goal_prediction_module = None  # GoalPredictionModule(total_emb_size=32)
        else:
            self.goal_prediction_module = None

        self.final_module = Unet5Contextual(in_channels=num_channels,
                                            out_channels=1,
                                            embedding_size=256)

        if torch.cuda.is_available():
            self.image_module.cuda()
            self.text_module.cuda()
            self.final_module.cuda()
            if self.object_detection_module is not None:
                self.object_detection_module.cuda()
            if self.goal_prediction_module is not None:
                self.goal_prediction_module.cuda()
Beispiel #6
0
    def __init__(self, config, constants):
        AbstractIncrementalModel.__init__(self, config, constants)
        self.none_action = config["num_actions"]

        self.config = config
        self.constants = constants

        # CNN over images - using what is essentially SimpleImage currently
        self.image_module = ImageCnnEmnlp(
            image_emb_size=constants["image_emb_dim"],
            input_num_channels=3 *
            5,  # 3 channels per image - 5 images in history
            image_height=config["image_height"],
            image_width=config["image_width"])

        # LSTM to embed text
        self.text_module = TextSimpleModule(
            emb_dim=constants["word_emb_dim"],
            hidden_dim=constants["lstm_emb_dim"],
            vocab_size=config["vocab_size"])

        # Action module to embed previous action+block
        self.action_module = ActionSimpleModule(
            num_actions=config["num_actions"],
            action_emb_size=constants["action_emb_dim"])

        # Put it all together
        self.final_module = IncrementalMultimodalEmnlp(
            image_module=self.image_module,
            text_module=self.text_module,
            action_module=self.action_module,
            input_embedding_size=constants["lstm_emb_dim"] +
            constants["image_emb_dim"] + constants["action_emb_dim"],
            output_hidden_size=config["h1_hidden_dim"],
            blocks_hidden_size=config["blocks_hidden_dim"],
            directions_hidden_size=config["action_hidden_dim"],
            max_episode_length=(constants["horizon"] + 5))

        if torch.cuda.is_available():
            self.image_module.cuda()
            self.text_module.cuda()
            self.action_module.cuda()
            self.final_module.cuda()
    def __init__(self,
                 config,
                 constants,
                 final_model_type="unet",
                 final_dimension=None):
        AbstractIncrementalModel.__init__(self, config, constants)
        self.none_action = config["num_actions"]
        self.image_module = UnetImageModule(
            image_emb_size=constants["image_emb_dim"],
            input_num_channels=3,
            image_height=config["image_height"],
            image_width=config["image_width"],
            using_recurrence=True,
            final_dimension=final_dimension)
        num_channels, image_height, image_width = self.image_module.get_final_dimension(
        )
        self.num_cameras = 1
        self.image_recurrence_module = IncrementalRecurrenceChaplotModule(
            input_emb_dim=256, output_emb_dim=256)
        if config["use_pointer_model"]:
            raise NotImplementedError()
        else:
            self.text_module = ChaplotTextModule(
                emb_dim=32,
                hidden_dim=256,
                vocab_size=config["vocab_size"],
                image_height=image_height,
                image_width=image_width)

        if config["do_action_prediction"]:
            self.action_prediction_module = ActionPredictionModule(
                2 * self.num_cameras * constants["image_emb_dim"],
                constants["image_emb_dim"], config["num_actions"])
        else:
            self.action_prediction_module = None

        if config["do_temporal_autoencoding"]:
            self.temporal_autoencoder_module = TemporalAutoencoderModule(
                self.action_module,
                self.num_cameras * constants["image_emb_dim"],
                constants["action_emb_dim"], constants["image_emb_dim"])
        else:
            self.temporal_autoencoder_module = None

        if config["do_object_detection"]:
            self.landmark_names = get_all_landmark_names()
            self.object_detection_module = PixelIdentificationModule(
                num_channels=num_channels, num_objects=67)
        else:
            self.object_detection_module = None

        if config["do_symbolic_language_prediction"]:
            self.symbolic_language_prediction_module = SymbolicLanguagePredictionModule(
                total_emb_size=2 * constants["lstm_emb_dim"])
        else:
            self.symbolic_language_prediction_module = None

        if config["do_goal_prediction"]:
            self.goal_prediction_module = None  # GoalPredictionModule(total_emb_size=32)
        else:
            self.goal_prediction_module = None

        if final_model_type == "m4jksum1":
            self.final_module = IncrementalMultimodalAttentionChaplotModuleM4JKSUM1(
                image_module=self.image_module,
                image_recurrence_module=self.image_recurrence_module,
                text_module=self.text_module,
                max_episode_length=150,
                final_image_height=image_height,
                final_image_width=image_width)
        elif final_model_type == "unet":
            self.final_module = IncrementalUnetAttentionModuleJustProb(
                image_module=self.image_module,
                image_recurrence_module=self.image_recurrence_module,
                text_module=self.text_module,
                max_episode_length=150,
                final_image_height=image_height,
                final_image_width=image_width,
                in_channels=num_channels,
                out_channels=1,
                embedding_size=256)
        elif final_model_type == "unet-positional-encoding":
            self.final_module = IncrementalUnetAttentionModuleJustProbSpatialEncoding(
                image_module=self.image_module,
                image_recurrence_module=self.image_recurrence_module,
                text_module=self.text_module,
                max_episode_length=150,
                final_image_height=image_height,
                final_image_width=image_width,
                in_channels=num_channels,
                out_channels=1,
                embedding_size=256)
        elif final_model_type == "andrew":
            self.final_module = IncrementalMultimodalAttentionChaplotModuleM5AndrewV2(
                image_module=self.image_module,
                image_recurrence_module=self.image_recurrence_module,
                text_module=self.text_module,
                max_episode_length=150,
                final_image_height=image_height,
                final_image_width=image_width,
                normalize_filters=False)
        else:
            raise AssertionError("Unknown final model type ", final_model_type)
        if torch.cuda.is_available():
            self.image_module.cuda()
            self.image_recurrence_module.cuda()
            self.text_module.cuda()
            self.final_module.cuda()
            if self.action_prediction_module is not None:
                self.action_prediction_module.cuda()
            if self.temporal_autoencoder_module is not None:
                self.temporal_autoencoder_module.cuda()
            if self.object_detection_module is not None:
                self.object_detection_module.cuda()
            if self.symbolic_language_prediction_module is not None:
                self.symbolic_language_prediction_module.cuda()
            if self.goal_prediction_module is not None:
                self.goal_prediction_module.cuda()
    def __init__(self, config, constants):
        AbstractIncrementalModel.__init__(self, config, constants)
        self.none_action = config["num_actions"]
        self.image_module = ChaplotImageModule(
            image_emb_size=constants["image_emb_dim"],
            input_num_channels=3,  # TODO this value keeps changing.
            image_height=config["image_height"],
            image_width=config["image_width"],
            using_recurrence=True)
        self.num_cameras = 1
        self.image_recurrence_module = IncrementalRecurrenceChaplotModule(
            input_emb_dim=256,
            output_emb_dim=256)
        if config["use_pointer_model"]:
            raise NotImplementedError()
        else:
            self.text_module = ChaplotTextModule(
                emb_dim=32,
                hidden_dim=256,
                vocab_size=config["vocab_size"],
                image_height=3, image_width=3)  # TODO these 4, 4, are shaky and keep changing.

        if config["do_action_prediction"]:
            self.action_prediction_module = ActionPredictionModule(
                2 * self.num_cameras * constants["image_emb_dim"], constants["image_emb_dim"], config["num_actions"])
        else:
            self.action_prediction_module = None

        if config["do_temporal_autoencoding"]:
            self.temporal_autoencoder_module = TemporalAutoencoderModule(
                self.action_module, self.num_cameras * constants["image_emb_dim"],
                constants["action_emb_dim"], constants["image_emb_dim"])
        else:
            self.temporal_autoencoder_module = None

        if config["do_object_detection"]:
            self.landmark_names = get_all_landmark_names()
            self.object_detection_module = ObjectDetectionModule(
                image_module=self.image_module, image_emb_size=self.num_cameras * constants["image_emb_dim"], num_objects=67)
        else:
            self.object_detection_module = None

        if config["do_symbolic_language_prediction"]:
            self.symbolic_language_prediction_module = SymbolicLanguagePredictionModule(
                total_emb_size=2 * constants["lstm_emb_dim"])
        else:
            self.symbolic_language_prediction_module = None

        if config["do_goal_prediction"]:
            self.goal_prediction_module = GoalPredictionModule(
                total_emb_size=32)
        else:
            self.goal_prediction_module = None

        self.final_module = IncrementalMultimodalChaplotModule(
            image_module=self.image_module,
            image_recurrence_module=self.image_recurrence_module,
            text_module=self.text_module,
            max_episode_length=(constants["horizon"] + constants["max_extra_horizon"]),
            final_image_height=3, final_image_width=3)  # TODO these 4, 4, are shaky and keep changing.
        if torch.cuda.is_available():
            self.image_module.cuda()
            self.image_recurrence_module.cuda()
            self.text_module.cuda()
            self.final_module.cuda()
            if self.action_prediction_module is not None:
                self.action_prediction_module.cuda()
            if self.temporal_autoencoder_module is not None:
                self.temporal_autoencoder_module.cuda()
            if self.object_detection_module is not None:
                self.object_detection_module.cuda()
            if self.symbolic_language_prediction_module is not None:
                self.symbolic_language_prediction_module.cuda()
            if self.goal_prediction_module is not None:
                self.goal_prediction_module.cuda()
Beispiel #9
0
    def __init__(self, config, constants):
        AbstractIncrementalModel.__init__(self, config, constants)
        self.none_action = config["num_actions"]
        self.image_module = ImageResnetModule(
            image_emb_size=constants["image_emb_dim"],
            input_num_channels=3,
            image_height=config["image_height"],
            image_width=config["image_width"],
            using_recurrence=True)
        self.num_cameras = 1
        self.image_recurrence_module = IncrementalRecurrenceSimpleModule(
            input_emb_dim=(constants["image_emb_dim"] * self.num_cameras + constants["action_emb_dim"]),
            output_emb_dim=constants["image_emb_dim"])
        if config["use_pointer_model"]:
            self.text_module = TextPointerModule(
                emb_dim=constants["word_emb_dim"],
                hidden_dim=constants["lstm_emb_dim"],
                vocab_size=config["vocab_size"])
        else:
            self.text_module = TextBiLSTMModule(
                emb_dim=constants["word_emb_dim"],
                hidden_dim=constants["lstm_emb_dim"],
                vocab_size=config["vocab_size"])
        self.action_module = ActionSimpleModule(
            num_actions=config["num_actions"],
            action_emb_size=constants["action_emb_dim"])
        if config["use_pointer_model"]:
            total_emb_size = (constants["image_emb_dim"]
                              + 4 * constants["lstm_emb_dim"]
                              + constants["action_emb_dim"])
        else:
            total_emb_size = ((self.num_cameras + 1) * constants["image_emb_dim"]
                              + 2 * constants["lstm_emb_dim"]
                              + constants["action_emb_dim"])

        if config["do_action_prediction"]:
            self.action_prediction_module = ActionPredictionModule(
                2 * self.num_cameras * constants["image_emb_dim"], constants["image_emb_dim"], config["num_actions"])
        else:
            self.action_prediction_module = None

        if config["do_temporal_autoencoding"]:
            self.temporal_autoencoder_module = TemporalAutoencoderModule(
                self.action_module, self.num_cameras * constants["image_emb_dim"],
                constants["action_emb_dim"], constants["image_emb_dim"])
        else:
            self.temporal_autoencoder_module = None

        if config["do_object_detection"]:
            self.landmark_names = get_all_landmark_names()
            self.object_detection_module = ObjectDetectionModule(
                image_module=self.image_module, image_emb_size=self.num_cameras * constants["image_emb_dim"], num_objects=67)
        else:
            self.object_detection_module = None

        if config["do_symbolic_language_prediction"]:
            self.symbolic_language_prediction_module = SymbolicLanguagePredictionModule(
                total_emb_size=2 * constants["lstm_emb_dim"])
        else:
            self.symbolic_language_prediction_module = None

        if config["do_goal_prediction"]:
            self.goal_prediction_module = GoalPredictionModule(
                total_emb_size=32)
        else:
            self.goal_prediction_module = None

        final_module = TmpIncrementalMultimodalDenseValtsRecurrentSimpleModule(
            image_module=self.image_module,
            image_recurrence_module=self.image_recurrence_module,
            text_module=self.text_module,
            action_module=self.action_module,
            total_emb_size=total_emb_size,
            num_actions=config["num_actions"])
        self.final_module = final_module
        if torch.cuda.is_available():
            self.image_module.cuda()
            self.image_recurrence_module.cuda()
            self.text_module.cuda()
            self.action_module.cuda()
            self.final_module.cuda()
            if self.action_prediction_module is not None:
                self.action_prediction_module.cuda()
            if self.temporal_autoencoder_module is not None:
                self.temporal_autoencoder_module.cuda()
            if self.object_detection_module is not None:
                self.object_detection_module.cuda()
            if self.symbolic_language_prediction_module is not None:
                self.symbolic_language_prediction_module.cuda()
            if self.goal_prediction_module is not None:
                self.goal_prediction_module.cuda()
    def __init__(self, config, constants, use_image=False):
        AbstractIncrementalModel.__init__(self, config, constants)
        self.none_action = config["num_actions"]
        self.image_module = UnetImageModule(
            image_emb_size=constants["image_emb_dim"],
            input_num_channels=3,
            image_height=config["image_height"],
            image_width=config["image_width"],
            using_recurrence=True)
        num_channels, image_height, image_width = self.image_module.get_final_dimension()
        self.num_cameras = 1
        self.image_recurrence_module = IncrementalRecurrenceChaplotModule(
            input_emb_dim=256,
            output_emb_dim=256)
        if config["use_pointer_model"]:
            raise NotImplementedError()
        else:
            self.text_module = ChaplotTextModule(
                emb_dim=32,
                hidden_dim=256,
                vocab_size=config["vocab_size"],
                image_height=image_height, image_width=image_width)

        if config["do_action_prediction"]:
            self.action_prediction_module = ActionPredictionModule(
                2 * self.num_cameras * constants["image_emb_dim"], constants["image_emb_dim"], config["num_actions"])
        else:
            self.action_prediction_module = None

        if config["do_temporal_autoencoding"]:
            self.temporal_autoencoder_module = TemporalAutoencoderModule(
                self.action_module, self.num_cameras * constants["image_emb_dim"],
                constants["action_emb_dim"], constants["image_emb_dim"])
        else:
            self.temporal_autoencoder_module = None

        if config["do_object_detection"]:
            self.landmark_names = get_all_landmark_names()
            self.object_detection_module = PixelIdentificationModule(
                num_channels=num_channels, num_objects=67)
        else:
            self.object_detection_module = None

        if config["do_symbolic_language_prediction"]:
            self.symbolic_language_prediction_module = SymbolicLanguagePredictionModule(
                total_emb_size=2 * constants["lstm_emb_dim"])
        else:
            self.symbolic_language_prediction_module = None

        if config["do_goal_prediction"]:
            self.goal_prediction_module = None  # GoalPredictionModule(total_emb_size=32)
        else:
            self.goal_prediction_module = None

        if use_image:
            self.final_module = OracleGoldWithImage(
                image_recurrence_module=self.image_recurrence_module,
                image_module=self.image_module,
                max_episode_length=150,
                final_image_height=image_height, final_image_width=image_width)
        else:
            self.final_module = OracleGold(
                image_recurrence_module=self.image_recurrence_module,
                max_episode_length=150,
                final_image_height=image_height, final_image_width=image_width)

        if torch.cuda.is_available():
            self.image_module.cuda()
            self.image_recurrence_module.cuda()
            self.text_module.cuda()
            self.final_module.cuda()
            if self.action_prediction_module is not None:
                self.action_prediction_module.cuda()
            if self.temporal_autoencoder_module is not None:
                self.temporal_autoencoder_module.cuda()
            if self.object_detection_module is not None:
                self.object_detection_module.cuda()
            if self.symbolic_language_prediction_module is not None:
                self.symbolic_language_prediction_module.cuda()
            if self.goal_prediction_module is not None:
                self.goal_prediction_module.cuda()
Beispiel #11
0
    def __init__(self, config, constants):
        AbstractIncrementalModel.__init__(self, config, constants)
        self.none_action = config["num_actions"]
        landmark_names = get_all_landmark_names()
        self.radius_module = RadiusModule(15)
        self.angle_module = AngleModule(12)  # (48)
        self.landmark_module = LandmarkModule(67)
        self.num_cameras = 1
        self.image_module = ImageRyanResnetModule(
            image_emb_size=constants["image_emb_dim"],
            input_num_channels=3,
            image_height=config["image_height"],
            image_width=config["image_width"],
            using_recurrence=True)
        self.image_recurrence_module = IncrementalRecurrenceSimpleModule(
            input_emb_dim=constants["image_emb_dim"] * self.num_cameras, # + constants["action_emb_dim"],
            output_emb_dim=constants["image_emb_dim"])
        self.text_module = SymbolicInstructionModule(
            radius_embedding=self.radius_module,
            theta_embedding=self.angle_module,
            landmark_embedding=self.landmark_module)
        self.action_module = ActionSimpleModule(
            num_actions=config["num_actions"],
            action_emb_size=constants["action_emb_dim"])
        total_emb_size = ((self.num_cameras) * constants["image_emb_dim"]
                          + 32 * 2
                          + constants["action_emb_dim"])

        if config["do_action_prediction"]:
            self.action_prediction_module = ActionPredictionModule(
                2 * self.num_cameras * constants["image_emb_dim"], constants["image_emb_dim"], config["num_actions"])
        else:
            self.action_prediction_module = None

        if config["do_temporal_autoencoding"]:
            self.temporal_autoencoder_module = TemporalAutoencoderModule(
                self.action_module, self.num_cameras * constants["image_emb_dim"],
                constants["action_emb_dim"], constants["image_emb_dim"])
        else:
            self.temporal_autoencoder_module = None

        if config["do_object_detection"]:
            self.landmark_names = get_all_landmark_names()
            self.object_detection_module = ObjectDetectionModule(
                image_module=self.image_module,
                image_emb_size=self.num_cameras * constants["image_emb_dim"], num_objects=67)
        else:
            self.object_detection_module = None

        final_module = IncrementalMultimodalRecurrentSimpleModule(
            image_module=self.image_module,
            image_recurrence_module=self.image_recurrence_module,
            text_module=self.text_module,
            action_module=self.action_module,
            total_emb_size=total_emb_size,
            num_actions=config["num_actions"])
        self.final_module = final_module
        if torch.cuda.is_available():
            self.image_module.cuda()
            self.image_recurrence_module.cuda()
            self.text_module.cuda()
            self.action_module.cuda()
            self.final_module.cuda()
            if self.action_prediction_module is not None:
                self.action_prediction_module.cuda()
            if self.temporal_autoencoder_module is not None:
                self.temporal_autoencoder_module.cuda()
            if self.object_detection_module is not None:
                self.object_detection_module.cuda()
 def __init__(self, base_models, config, constants):
     AbstractIncrementalModel.__init__(self, config, constants)
     self.base_models = base_models
     self.config = config
     self.constants = constants