def __init__(self, params):
"""Creates class instance.
Arguments:
params
.env (gym.env): Environment for which this controller will be used.
.ac_ub (np.ndarray): (optional) An array of action upper bounds.
Defaults to environment action upper bounds.
.ac_lb (np.ndarray): (optional) An array of action lower bounds.
Defaults to environment action lower bounds.
.per (int): (optional) Determines how often the action sequence will be optimized.
Defaults to 1 (reoptimizes at every call to act()).
.opt_cfg
.cfg (DotMap): A map of optimizer initializer parameters.
.plan_hor (int): The planning horizon that will be used in optimization.
"""
super().__init__(params)
self.env = params.env
self.temp_env = params.temp_env
self.env_name = params.env_name
self.dU = params.env.action_space.shape[0]
self.ac_ub, self.ac_lb = params.env.action_space.high, params.env.action_space.low
self.ac_ub = np.minimum(self.ac_ub, params.get("ac_ub", self.ac_ub))
self.ac_lb = np.maximum(self.ac_lb, params.get("ac_lb", self.ac_lb))
# Create action sequence optimizer
opt_cfg = params.opt_cfg.get("cfg", {})
self.plan_hor = get_required_argument(
params.opt_cfg, "plan_hor", "Must provide planning horizon.")
self.popsize = opt_cfg['popsize']
self.num_elites = opt_cfg['num_elites']
self.max_iters = opt_cfg['max_iters']
self.alpha = opt_cfg['alpha']
self.prev_sol = np.tile((self.ac_lb + self.ac_ub) / 2, [self.plan_hor])
self.init_var = np.tile(
np.square(self.ac_ub - self.ac_lb) / 16, [self.plan_hor])
self.encoder = params.encoder
self.transition_model = params.transition_model
self.residual_model = params.residual_model
self.dynamics_optimizer = params.dynamics_optimizer
self.dynamics_finetune_optimizer = params.dynamics_finetune_optimizer
self.hidden_size = params.hidden_size
self.beta = params.beta
self.logdir = params.logdir
self.batch_size = params.batch_size
self.value_func = None
def nn_constructor(self, model_init_cfg):
ensemble_size = get_required_argument(model_init_cfg, "num_nets",
"Must provide ensemble size")
load_model = model_init_cfg.get("load_model", False)
assert load_model is False, 'Has yet to support loading model'
model = PtModel(ensemble_size, self.MODEL_IN,
self.MODEL_OUT * 2).to(TORCH_DEVICE)
# * 2 because we output both the mean and the variance
model.optim = torch.optim.Adam(model.parameters(), lr=0.001)
return model
def __init__(self, params):
"""Creates class instance.
Arguments:
params
.env (gym.env): Environment for which this controller will be used.
.ac_ub (np.ndarray): (optional) An array of action upper bounds.
Defaults to environment action upper bounds.
.ac_lb (np.ndarray): (optional) An array of action lower bounds.
Defaults to environment action lower bounds.
.per (int): (optional) Determines how often the action sequence will be optimized.
Defaults to 1 (reoptimizes at every call to act()).
.prop_cfg
.model_init_cfg (DotMap): A DotMap of initialization parameters for the model.
.model_constructor (func): A function which constructs an instance of this
model, given model_init_cfg.
.model_train_cfg (dict): (optional) A DotMap of training parameters that will be passed
into the model every time is is trained. Defaults to an empty dict.
.model_pretrained (bool): (optional) If True, assumes that the model
has been trained upon construction.
.mode (str): Propagation method. Choose between [E, DS, TSinf, TS1, MM].
See https://arxiv.org/abs/1805.12114 for details.
.npart (int): Number of particles used for DS, TSinf, TS1, and MM propagation methods.
.ign_var (bool): (optional) Determines whether or not variance output of the model
will be ignored. Defaults to False unless deterministic propagation is being used.
.obs_preproc (func): (optional) A function which modifies observations (in a 2D matrix)
before they are passed into the model. Defaults to lambda obs: obs.
Note: Must be able to process both NumPy and Tensorflow arrays.
.obs_postproc (func): (optional) A function which returns vectors calculated from
the previous observations and model predictions, which will then be passed into
the provided cost function on observations. Defaults to lambda obs, model_out: model_out.
Note: Must be able to process both NumPy and Tensorflow arrays.
.obs_postproc2 (func): (optional) A function which takes the vectors returned by
obs_postproc and (possibly) modifies it into the predicted observations for the
next time step. Defaults to lambda obs: obs.
Note: Must be able to process both NumPy and Tensorflow arrays.
.targ_proc (func): (optional) A function which takes current observations and next
observations and returns the array of targets (so that the model learns the mapping
obs -> targ_proc(obs, next_obs)). Defaults to lambda obs, next_obs: next_obs.
Note: Only needs to process NumPy arrays.
.opt_cfg
.mode (str): Internal optimizer that will be used. Choose between [CEM].
.cfg (DotMap): A map of optimizer initializer parameters.
.plan_hor (int): The planning horizon that will be used in optimization.
.obs_cost_fn (func): A function which computes the cost of every observation
in a 2D matrix.
Note: Must be able to process both NumPy and Tensorflow arrays.
.ac_cost_fn (func): A function which computes the cost of every action
in a 2D matrix.
.log_cfg
.save_all_models (bool): (optional) If True, saves models at every iteration.
Defaults to False (only most recent model is saved).
Warning: Can be very memory-intensive.
.log_traj_preds (bool): (optional) If True, saves the mean and variance of predicted
particle trajectories. Defaults to False.
.log_particles (bool) (optional) If True, saves all predicted particles trajectories.
Defaults to False. Note: Takes precedence over log_traj_preds.
Warning: Can be very memory-intensive
"""
super().__init__(params)
self.dO, self.dU = params.env.observation_space.shape[
0], params.env.action_space.shape[0]
self.ac_ub, self.ac_lb = params.env.action_space.high, params.env.action_space.low
self.ac_ub = np.minimum(self.ac_ub, params.get("ac_ub", self.ac_ub))
self.ac_lb = np.maximum(self.ac_lb, params.get("ac_lb", self.ac_lb))
self.update_fns = params.get("update_fns", [])
self.per = params.get("per", 1)
self.model_init_cig = params.prop_cfg.get("model_init_cfg", {})
self.model_train_cfg = params.prop_cfg.get("model_train_cfg", {})
self.prop_mode = get_required_argument(
params.prop_cfg, "mode", "Must provide propagation method.")
self.npart = get_required_argument(
params.prop_cfg, "npart", "Must provide number of particles.")
self.ign_var = params.prop_cfg.get("ign_var",
False) or self.prop_mode == "E"
self.obs_preproc = params.prop_cfg.get("obs_preproc", lambda obs: obs)
self.obs_postproc = params.prop_cfg.get(
"obs_postproc", lambda obs, model_out: model_out)
self.obs_postproc2 = params.prop_cfg.get("obs_postproc2",
lambda next_obs: next_obs)
self.targ_proc = params.prop_cfg.get("targ_proc",
lambda obs, next_obs: next_obs)
self.opt_mode = get_required_argument(
params.opt_cfg, "mode", "Must provide optimization method.")
self.plan_hor = get_required_argument(
params.opt_cfg, "plan_hor", "Must provide planning horizon.")
self.obs_cost_fn = get_required_argument(
params.opt_cfg, "obs_cost_fn",
"Must provide cost on observations.")
self.ac_cost_fn = get_required_argument(
params.opt_cfg, "ac_cost_fn", "Must provide cost on actions.")
self.save_all_models = params.log_cfg.get("save_all_models", False)
self.log_traj_preds = params.log_cfg.get("log_traj_preds", False)
self.log_particles = params.log_cfg.get("log_particles", False)
# Perform argument checks
assert self.opt_mode == 'CEM'
assert self.prop_mode == 'TSinf', 'only TSinf propagation mode is supported'
assert self.npart % self.model_init_cig.num_nets == 0, "Number of particles must be a multiple of the ensemble size."
# Create action sequence optimizer
opt_cfg = params.opt_cfg.get("cfg", {})
self.optimizer = CEMOptimizer(
sol_dim=self.plan_hor * self.dU,
lower_bound=np.tile(self.ac_lb, [self.plan_hor]),
upper_bound=np.tile(self.ac_ub, [self.plan_hor]),
cost_function=self._compile_cost,
**opt_cfg)
# Controller state variables
self.has_been_trained = params.prop_cfg.get("model_pretrained", False)
self.ac_buf = np.array([]).reshape(0, self.dU)
self.prev_sol = np.tile((self.ac_lb + self.ac_ub) / 2, [self.plan_hor])
self.init_var = np.tile(
np.square(self.ac_ub - self.ac_lb) / 16, [self.plan_hor])
self.train_in = np.array([]).reshape(
0, self.dU + self.obs_preproc(np.zeros([1, self.dO])).shape[-1])
self.train_targs = np.array([]).reshape(
0,
self.targ_proc(np.zeros([1, self.dO]),
np.zeros([1, self.dO])).shape[-1])
print("Created an MPC controller, prop mode %s, %d particles. " %
(self.prop_mode, self.npart) +
("Ignoring variance." if self.ign_var else ""))
if self.save_all_models:
print(
"Controller will save all models. (Note: This may be memory-intensive."
)
if self.log_particles:
print(
"Controller is logging particle predictions (Note: This may be memory-intensive)."
)
self.pred_particles = []
elif self.log_traj_preds:
print(
"Controller is logging trajectory prediction statistics (mean+var)."
)
self.pred_means, self.pred_vars = [], []
else:
print("Trajectory prediction logging is disabled.")
# Set up pytorch model
self.model = get_required_argument(
params.prop_cfg.model_init_cfg, "model_constructor",
"Must provide a model constructor.")(
params.prop_cfg.model_init_cfg)
self.value_func = None