def __init__(self, params): """Creates class instance. Arguments: params .env (gym.env): Environment for which this controller will be used. .ac_ub (np.ndarray): (optional) An array of action upper bounds. Defaults to environment action upper bounds. .ac_lb (np.ndarray): (optional) An array of action lower bounds. Defaults to environment action lower bounds. .per (int): (optional) Determines how often the action sequence will be optimized. Defaults to 1 (reoptimizes at every call to act()). .opt_cfg .cfg (DotMap): A map of optimizer initializer parameters. .plan_hor (int): The planning horizon that will be used in optimization. """ super().__init__(params) self.env = params.env self.temp_env = params.temp_env self.env_name = params.env_name self.dU = params.env.action_space.shape[0] self.ac_ub, self.ac_lb = params.env.action_space.high, params.env.action_space.low self.ac_ub = np.minimum(self.ac_ub, params.get("ac_ub", self.ac_ub)) self.ac_lb = np.maximum(self.ac_lb, params.get("ac_lb", self.ac_lb)) # Create action sequence optimizer opt_cfg = params.opt_cfg.get("cfg", {}) self.plan_hor = get_required_argument( params.opt_cfg, "plan_hor", "Must provide planning horizon.") self.popsize = opt_cfg['popsize'] self.num_elites = opt_cfg['num_elites'] self.max_iters = opt_cfg['max_iters'] self.alpha = opt_cfg['alpha'] self.prev_sol = np.tile((self.ac_lb + self.ac_ub) / 2, [self.plan_hor]) self.init_var = np.tile( np.square(self.ac_ub - self.ac_lb) / 16, [self.plan_hor]) self.encoder = params.encoder self.transition_model = params.transition_model self.residual_model = params.residual_model self.dynamics_optimizer = params.dynamics_optimizer self.dynamics_finetune_optimizer = params.dynamics_finetune_optimizer self.hidden_size = params.hidden_size self.beta = params.beta self.logdir = params.logdir self.batch_size = params.batch_size self.value_func = None
def nn_constructor(self, model_init_cfg): ensemble_size = get_required_argument(model_init_cfg, "num_nets", "Must provide ensemble size") load_model = model_init_cfg.get("load_model", False) assert load_model is False, 'Has yet to support loading model' model = PtModel(ensemble_size, self.MODEL_IN, self.MODEL_OUT * 2).to(TORCH_DEVICE) # * 2 because we output both the mean and the variance model.optim = torch.optim.Adam(model.parameters(), lr=0.001) return model
def __init__(self, params): """Creates class instance. Arguments: params .env (gym.env): Environment for which this controller will be used. .ac_ub (np.ndarray): (optional) An array of action upper bounds. Defaults to environment action upper bounds. .ac_lb (np.ndarray): (optional) An array of action lower bounds. Defaults to environment action lower bounds. .per (int): (optional) Determines how often the action sequence will be optimized. Defaults to 1 (reoptimizes at every call to act()). .prop_cfg .model_init_cfg (DotMap): A DotMap of initialization parameters for the model. .model_constructor (func): A function which constructs an instance of this model, given model_init_cfg. .model_train_cfg (dict): (optional) A DotMap of training parameters that will be passed into the model every time is is trained. Defaults to an empty dict. .model_pretrained (bool): (optional) If True, assumes that the model has been trained upon construction. .mode (str): Propagation method. Choose between [E, DS, TSinf, TS1, MM]. See https://arxiv.org/abs/1805.12114 for details. .npart (int): Number of particles used for DS, TSinf, TS1, and MM propagation methods. .ign_var (bool): (optional) Determines whether or not variance output of the model will be ignored. Defaults to False unless deterministic propagation is being used. .obs_preproc (func): (optional) A function which modifies observations (in a 2D matrix) before they are passed into the model. Defaults to lambda obs: obs. Note: Must be able to process both NumPy and Tensorflow arrays. .obs_postproc (func): (optional) A function which returns vectors calculated from the previous observations and model predictions, which will then be passed into the provided cost function on observations. Defaults to lambda obs, model_out: model_out. Note: Must be able to process both NumPy and Tensorflow arrays. .obs_postproc2 (func): (optional) A function which takes the vectors returned by obs_postproc and (possibly) modifies it into the predicted observations for the next time step. Defaults to lambda obs: obs. Note: Must be able to process both NumPy and Tensorflow arrays. .targ_proc (func): (optional) A function which takes current observations and next observations and returns the array of targets (so that the model learns the mapping obs -> targ_proc(obs, next_obs)). Defaults to lambda obs, next_obs: next_obs. Note: Only needs to process NumPy arrays. .opt_cfg .mode (str): Internal optimizer that will be used. Choose between [CEM]. .cfg (DotMap): A map of optimizer initializer parameters. .plan_hor (int): The planning horizon that will be used in optimization. .obs_cost_fn (func): A function which computes the cost of every observation in a 2D matrix. Note: Must be able to process both NumPy and Tensorflow arrays. .ac_cost_fn (func): A function which computes the cost of every action in a 2D matrix. .log_cfg .save_all_models (bool): (optional) If True, saves models at every iteration. Defaults to False (only most recent model is saved). Warning: Can be very memory-intensive. .log_traj_preds (bool): (optional) If True, saves the mean and variance of predicted particle trajectories. Defaults to False. .log_particles (bool) (optional) If True, saves all predicted particles trajectories. Defaults to False. Note: Takes precedence over log_traj_preds. Warning: Can be very memory-intensive """ super().__init__(params) self.dO, self.dU = params.env.observation_space.shape[ 0], params.env.action_space.shape[0] self.ac_ub, self.ac_lb = params.env.action_space.high, params.env.action_space.low self.ac_ub = np.minimum(self.ac_ub, params.get("ac_ub", self.ac_ub)) self.ac_lb = np.maximum(self.ac_lb, params.get("ac_lb", self.ac_lb)) self.update_fns = params.get("update_fns", []) self.per = params.get("per", 1) self.model_init_cig = params.prop_cfg.get("model_init_cfg", {}) self.model_train_cfg = params.prop_cfg.get("model_train_cfg", {}) self.prop_mode = get_required_argument( params.prop_cfg, "mode", "Must provide propagation method.") self.npart = get_required_argument( params.prop_cfg, "npart", "Must provide number of particles.") self.ign_var = params.prop_cfg.get("ign_var", False) or self.prop_mode == "E" self.obs_preproc = params.prop_cfg.get("obs_preproc", lambda obs: obs) self.obs_postproc = params.prop_cfg.get( "obs_postproc", lambda obs, model_out: model_out) self.obs_postproc2 = params.prop_cfg.get("obs_postproc2", lambda next_obs: next_obs) self.targ_proc = params.prop_cfg.get("targ_proc", lambda obs, next_obs: next_obs) self.opt_mode = get_required_argument( params.opt_cfg, "mode", "Must provide optimization method.") self.plan_hor = get_required_argument( params.opt_cfg, "plan_hor", "Must provide planning horizon.") self.obs_cost_fn = get_required_argument( params.opt_cfg, "obs_cost_fn", "Must provide cost on observations.") self.ac_cost_fn = get_required_argument( params.opt_cfg, "ac_cost_fn", "Must provide cost on actions.") self.save_all_models = params.log_cfg.get("save_all_models", False) self.log_traj_preds = params.log_cfg.get("log_traj_preds", False) self.log_particles = params.log_cfg.get("log_particles", False) # Perform argument checks assert self.opt_mode == 'CEM' assert self.prop_mode == 'TSinf', 'only TSinf propagation mode is supported' assert self.npart % self.model_init_cig.num_nets == 0, "Number of particles must be a multiple of the ensemble size." # Create action sequence optimizer opt_cfg = params.opt_cfg.get("cfg", {}) self.optimizer = CEMOptimizer( sol_dim=self.plan_hor * self.dU, lower_bound=np.tile(self.ac_lb, [self.plan_hor]), upper_bound=np.tile(self.ac_ub, [self.plan_hor]), cost_function=self._compile_cost, **opt_cfg) # Controller state variables self.has_been_trained = params.prop_cfg.get("model_pretrained", False) self.ac_buf = np.array([]).reshape(0, self.dU) self.prev_sol = np.tile((self.ac_lb + self.ac_ub) / 2, [self.plan_hor]) self.init_var = np.tile( np.square(self.ac_ub - self.ac_lb) / 16, [self.plan_hor]) self.train_in = np.array([]).reshape( 0, self.dU + self.obs_preproc(np.zeros([1, self.dO])).shape[-1]) self.train_targs = np.array([]).reshape( 0, self.targ_proc(np.zeros([1, self.dO]), np.zeros([1, self.dO])).shape[-1]) print("Created an MPC controller, prop mode %s, %d particles. " % (self.prop_mode, self.npart) + ("Ignoring variance." if self.ign_var else "")) if self.save_all_models: print( "Controller will save all models. (Note: This may be memory-intensive." ) if self.log_particles: print( "Controller is logging particle predictions (Note: This may be memory-intensive)." ) self.pred_particles = [] elif self.log_traj_preds: print( "Controller is logging trajectory prediction statistics (mean+var)." ) self.pred_means, self.pred_vars = [], [] else: print("Trajectory prediction logging is disabled.") # Set up pytorch model self.model = get_required_argument( params.prop_cfg.model_init_cfg, "model_constructor", "Must provide a model constructor.")( params.prop_cfg.model_init_cfg) self.value_func = None