def __init__(self, config, constants, final_model_type="unet", final_dimension=None): AbstractIncrementalModel.__init__(self, config, constants) self.none_action = config["num_actions"] self.image_module = UnetImageModule( image_emb_size=constants["image_emb_dim"], input_num_channels=3, image_height=config["image_height"], image_width=config["image_width"], using_recurrence=True, final_dimension=final_dimension) num_channels, image_height, image_width = self.image_module.get_final_dimension( ) self.num_cameras = 1 self.image_recurrence_module = IncrementalRecurrenceChaplotModule( input_emb_dim=256, output_emb_dim=256) if config["use_pointer_model"]: raise NotImplementedError() else: self.text_module = ChaplotTextModule( emb_dim=32, hidden_dim=256, vocab_size=config["vocab_size"], image_height=image_height, image_width=image_width) if config["do_action_prediction"]: self.action_prediction_module = ActionPredictionModule( 2 * self.num_cameras * constants["image_emb_dim"], constants["image_emb_dim"], config["num_actions"]) else: self.action_prediction_module = None if config["do_temporal_autoencoding"]: self.temporal_autoencoder_module = TemporalAutoencoderModule( self.action_module, self.num_cameras * constants["image_emb_dim"], constants["action_emb_dim"], constants["image_emb_dim"]) else: self.temporal_autoencoder_module = None if config["do_object_detection"]: self.landmark_names = get_all_landmark_names() self.object_detection_module = PixelIdentificationModule( num_channels=num_channels, num_objects=67) else: self.object_detection_module = None if config["do_symbolic_language_prediction"]: self.symbolic_language_prediction_module = SymbolicLanguagePredictionModule( total_emb_size=2 * constants["lstm_emb_dim"]) else: self.symbolic_language_prediction_module = None if config["do_goal_prediction"]: self.goal_prediction_module = None # GoalPredictionModule(total_emb_size=32) else: self.goal_prediction_module = None if final_model_type == "m4jksum1": self.final_module = IncrementalMultimodalAttentionChaplotModuleM4JKSUM1( image_module=self.image_module, image_recurrence_module=self.image_recurrence_module, text_module=self.text_module, max_episode_length=150, final_image_height=image_height, final_image_width=image_width) elif final_model_type == "unet": self.final_module = IncrementalUnetAttentionModuleJustProb( image_module=self.image_module, image_recurrence_module=self.image_recurrence_module, text_module=self.text_module, max_episode_length=150, final_image_height=image_height, final_image_width=image_width, in_channels=num_channels, out_channels=1, embedding_size=256) elif final_model_type == "unet-positional-encoding": self.final_module = IncrementalUnetAttentionModuleJustProbSpatialEncoding( image_module=self.image_module, image_recurrence_module=self.image_recurrence_module, text_module=self.text_module, max_episode_length=150, final_image_height=image_height, final_image_width=image_width, in_channels=num_channels, out_channels=1, embedding_size=256) elif final_model_type == "andrew": self.final_module = IncrementalMultimodalAttentionChaplotModuleM5AndrewV2( image_module=self.image_module, image_recurrence_module=self.image_recurrence_module, text_module=self.text_module, max_episode_length=150, final_image_height=image_height, final_image_width=image_width, normalize_filters=False) else: raise AssertionError("Unknown final model type ", final_model_type) if torch.cuda.is_available(): self.image_module.cuda() self.image_recurrence_module.cuda() self.text_module.cuda() self.final_module.cuda() if self.action_prediction_module is not None: self.action_prediction_module.cuda() if self.temporal_autoencoder_module is not None: self.temporal_autoencoder_module.cuda() if self.object_detection_module is not None: self.object_detection_module.cuda() if self.symbolic_language_prediction_module is not None: self.symbolic_language_prediction_module.cuda() if self.goal_prediction_module is not None: self.goal_prediction_module.cuda()
def __init__(self, config, constants): AbstractIncrementalModel.__init__(self, config, constants) self.none_action = config["num_actions"] self.image_module = ImageResnetModule( image_emb_size=constants["image_emb_dim"], input_num_channels=3, image_height=config["image_height"], image_width=config["image_width"], using_recurrence=True) self.num_cameras = 1 self.image_recurrence_module = IncrementalRecurrenceSimpleModule( input_emb_dim=(constants["image_emb_dim"] * self.num_cameras + constants["action_emb_dim"]), output_emb_dim=constants["image_emb_dim"]) if config["use_pointer_model"]: self.text_module = TextPointerModule( emb_dim=constants["word_emb_dim"], hidden_dim=constants["lstm_emb_dim"], vocab_size=config["vocab_size"]) else: self.text_module = TextBiLSTMModule( emb_dim=constants["word_emb_dim"], hidden_dim=constants["lstm_emb_dim"], vocab_size=config["vocab_size"]) self.action_module = ActionSimpleModule( num_actions=config["num_actions"], action_emb_size=constants["action_emb_dim"]) if config["use_pointer_model"]: total_emb_size = (constants["image_emb_dim"] + 4 * constants["lstm_emb_dim"] + constants["action_emb_dim"]) else: total_emb_size = ((self.num_cameras + 1) * constants["image_emb_dim"] + 2 * constants["lstm_emb_dim"] + constants["action_emb_dim"]) if config["do_action_prediction"]: self.action_prediction_module = ActionPredictionModule( 2 * self.num_cameras * constants["image_emb_dim"], constants["image_emb_dim"], config["num_actions"]) else: self.action_prediction_module = None if config["do_temporal_autoencoding"]: self.temporal_autoencoder_module = TemporalAutoencoderModule( self.action_module, self.num_cameras * constants["image_emb_dim"], constants["action_emb_dim"], constants["image_emb_dim"]) else: self.temporal_autoencoder_module = None if config["do_object_detection"]: self.landmark_names = get_all_landmark_names() self.object_detection_module = ObjectDetectionModule( image_module=self.image_module, image_emb_size=self.num_cameras * constants["image_emb_dim"], num_objects=67) else: self.object_detection_module = None if config["do_symbolic_language_prediction"]: self.symbolic_language_prediction_module = SymbolicLanguagePredictionModule( total_emb_size=2 * constants["lstm_emb_dim"]) else: self.symbolic_language_prediction_module = None if config["do_goal_prediction"]: self.goal_prediction_module = GoalPredictionModule( total_emb_size=32) else: self.goal_prediction_module = None final_module = TmpIncrementalMultimodalDenseValtsRecurrentSimpleModule( image_module=self.image_module, image_recurrence_module=self.image_recurrence_module, text_module=self.text_module, action_module=self.action_module, total_emb_size=total_emb_size, num_actions=config["num_actions"]) self.final_module = final_module if torch.cuda.is_available(): self.image_module.cuda() self.image_recurrence_module.cuda() self.text_module.cuda() self.action_module.cuda() self.final_module.cuda() if self.action_prediction_module is not None: self.action_prediction_module.cuda() if self.temporal_autoencoder_module is not None: self.temporal_autoencoder_module.cuda() if self.object_detection_module is not None: self.object_detection_module.cuda() if self.symbolic_language_prediction_module is not None: self.symbolic_language_prediction_module.cuda() if self.goal_prediction_module is not None: self.goal_prediction_module.cuda()
def __init__(self, config, constants): AbstractIncrementalModel.__init__(self, config, constants) self.none_action = config["num_actions"] self.image_module = ChaplotImageModule( image_emb_size=constants["image_emb_dim"], input_num_channels=3, # TODO this value keeps changing. image_height=config["image_height"], image_width=config["image_width"], using_recurrence=True) self.num_cameras = 1 self.image_recurrence_module = IncrementalRecurrenceChaplotModule( input_emb_dim=256, output_emb_dim=256) if config["use_pointer_model"]: raise NotImplementedError() else: self.text_module = ChaplotTextModule( emb_dim=32, hidden_dim=256, vocab_size=config["vocab_size"], image_height=3, image_width=3) # TODO these 4, 4, are shaky and keep changing. if config["do_action_prediction"]: self.action_prediction_module = ActionPredictionModule( 2 * self.num_cameras * constants["image_emb_dim"], constants["image_emb_dim"], config["num_actions"]) else: self.action_prediction_module = None if config["do_temporal_autoencoding"]: self.temporal_autoencoder_module = TemporalAutoencoderModule( self.action_module, self.num_cameras * constants["image_emb_dim"], constants["action_emb_dim"], constants["image_emb_dim"]) else: self.temporal_autoencoder_module = None if config["do_object_detection"]: self.landmark_names = get_all_landmark_names() self.object_detection_module = ObjectDetectionModule( image_module=self.image_module, image_emb_size=self.num_cameras * constants["image_emb_dim"], num_objects=67) else: self.object_detection_module = None if config["do_symbolic_language_prediction"]: self.symbolic_language_prediction_module = SymbolicLanguagePredictionModule( total_emb_size=2 * constants["lstm_emb_dim"]) else: self.symbolic_language_prediction_module = None if config["do_goal_prediction"]: self.goal_prediction_module = GoalPredictionModule( total_emb_size=32) else: self.goal_prediction_module = None self.final_module = IncrementalMultimodalChaplotModule( image_module=self.image_module, image_recurrence_module=self.image_recurrence_module, text_module=self.text_module, max_episode_length=(constants["horizon"] + constants["max_extra_horizon"]), final_image_height=3, final_image_width=3) # TODO these 4, 4, are shaky and keep changing. if torch.cuda.is_available(): self.image_module.cuda() self.image_recurrence_module.cuda() self.text_module.cuda() self.final_module.cuda() if self.action_prediction_module is not None: self.action_prediction_module.cuda() if self.temporal_autoencoder_module is not None: self.temporal_autoencoder_module.cuda() if self.object_detection_module is not None: self.object_detection_module.cuda() if self.symbolic_language_prediction_module is not None: self.symbolic_language_prediction_module.cuda() if self.goal_prediction_module is not None: self.goal_prediction_module.cuda()
def __init__(self, config, constants, use_image=False): AbstractIncrementalModel.__init__(self, config, constants) self.none_action = config["num_actions"] self.image_module = UnetImageModule( image_emb_size=constants["image_emb_dim"], input_num_channels=3, image_height=config["image_height"], image_width=config["image_width"], using_recurrence=True) num_channels, image_height, image_width = self.image_module.get_final_dimension() self.num_cameras = 1 self.image_recurrence_module = IncrementalRecurrenceChaplotModule( input_emb_dim=256, output_emb_dim=256) if config["use_pointer_model"]: raise NotImplementedError() else: self.text_module = ChaplotTextModule( emb_dim=32, hidden_dim=256, vocab_size=config["vocab_size"], image_height=image_height, image_width=image_width) if config["do_action_prediction"]: self.action_prediction_module = ActionPredictionModule( 2 * self.num_cameras * constants["image_emb_dim"], constants["image_emb_dim"], config["num_actions"]) else: self.action_prediction_module = None if config["do_temporal_autoencoding"]: self.temporal_autoencoder_module = TemporalAutoencoderModule( self.action_module, self.num_cameras * constants["image_emb_dim"], constants["action_emb_dim"], constants["image_emb_dim"]) else: self.temporal_autoencoder_module = None if config["do_object_detection"]: self.landmark_names = get_all_landmark_names() self.object_detection_module = PixelIdentificationModule( num_channels=num_channels, num_objects=67) else: self.object_detection_module = None if config["do_symbolic_language_prediction"]: self.symbolic_language_prediction_module = SymbolicLanguagePredictionModule( total_emb_size=2 * constants["lstm_emb_dim"]) else: self.symbolic_language_prediction_module = None if config["do_goal_prediction"]: self.goal_prediction_module = None # GoalPredictionModule(total_emb_size=32) else: self.goal_prediction_module = None if use_image: self.final_module = OracleGoldWithImage( image_recurrence_module=self.image_recurrence_module, image_module=self.image_module, max_episode_length=150, final_image_height=image_height, final_image_width=image_width) else: self.final_module = OracleGold( image_recurrence_module=self.image_recurrence_module, max_episode_length=150, final_image_height=image_height, final_image_width=image_width) if torch.cuda.is_available(): self.image_module.cuda() self.image_recurrence_module.cuda() self.text_module.cuda() self.final_module.cuda() if self.action_prediction_module is not None: self.action_prediction_module.cuda() if self.temporal_autoencoder_module is not None: self.temporal_autoencoder_module.cuda() if self.object_detection_module is not None: self.object_detection_module.cuda() if self.symbolic_language_prediction_module is not None: self.symbolic_language_prediction_module.cuda() if self.goal_prediction_module is not None: self.goal_prediction_module.cuda()