def setup_models(args: argparse.Namespace, env: Env) -> Tuple[Tuple[nn.Module, nn.Module, nn.Module, nn.Module],
Tuple[optim.Optimizer, optim.Optimizer],
List[torch.nn.parameter.Parameter]]:
# Initialise model parameters randomly
transition_model = TransitionModel(
args.belief_size,
args.state_size,
env.action_size,
args.hidden_size,
args.embedding_size,
args.activation_function
).to(device=args.device)
observation_model = ObservationDecoder(
args.belief_size,
args.state_size,
args.embedding_size
).to(device=args.device)
reward_model = RewardModel(
args.belief_size,
args.state_size,
args.hidden_size
).to(device=args.device)
encoder = ObservationEncoder(
args.embedding_size,
).to(device=args.device)
param_list = (
list(transition_model.parameters()) +
list(observation_model.parameters()) +
list(encoder.parameters())
)
transition_optimizer = optim.Adam(param_list, args.learning_rate, eps=args.adam_epsilon)
reward_optimizer = optim.Adam(reward_model.parameters(), args.learning_rate, eps=args.adam_epsilon)
# load parameters
if args.checkpoint_path is not None:
if os.path.exists(args.checkpoint_path):
model_dicts = torch.load(args.checkpoint_path)
transition_model.load_state_dict(model_dicts['transition_model'])
observation_model.load_state_dict(model_dicts['observation_model'])
reward_model.load_state_dict(model_dicts['reward_model'])
encoder.load_state_dict(model_dicts['encoder'])
transition_optimizer.load_state_dict(model_dicts['transition_optimizer'])
reward_optimizer.load_state_dict(model_dicts['reward_optimizer'])
else:
logging.warning("Model weight file: {} does not exist".format(args.checkpoint_path))
return (transition_model, observation_model, reward_model, encoder), (transition_optimizer, reward_optimizer), param_list
args.embedding_size, args.activation_function).to(device=args.device)
observation_model = ObservationModel(
args.symbolic_env, env.observation_size, args.belief_size, args.state_size,
args.embedding_size, args.activation_function).to(device=args.device)
reward_model = RewardModel(args.belief_size, args.state_size, args.hidden_size,
args.activation_function).to(device=args.device)
encoder = Encoder(args.symbolic_env, env.observation_size, args.embedding_size,
args.activation_function).to(device=args.device)
param_list = list(transition_model.parameters()) + list(
observation_model.parameters()) + list(reward_model.parameters()) + list(
encoder.parameters())
optimiser = optim.Adam(param_list, lr=args.learning_rate, eps=1e-4)
if args.load_checkpoint > 0:
model_dicts = torch.load(
os.path.join(results_dir, 'models_%d.pth' % args.load_checkpoint))
transition_model.load_state_dict(model_dicts['transition_model'])
observation_model.load_state_dict(model_dicts['observation_model'])
reward_model.load_state_dict(model_dicts['reward_model'])
encoder.load_state_dict(model_dicts['encoder'])
optimiser.load_state_dict(model_dicts['optimiser'])
mode = "continuous"
num_actions = -1
if type(env._env.action_space) == gym.spaces.discrete.Discrete:
mode = "discrete"
num_actions = env._env.action_space.n
planner = MPCPlanner(env.action_size,
args.planning_horizon,
args.optimisation_iters,
args.candidates,
args.top_candidates,
class Trainer():
def __init__(self, params, experience_replay_buffer,metrics,results_dir,env):
self.parms = params
self.D = experience_replay_buffer
self.metrics = metrics
self.env = env
self.tested_episodes = 0
self.statistics_path = results_dir+'/statistics'
self.model_path = results_dir+'/model'
self.video_path = results_dir+'/video'
self.rew_vs_pred_rew_path = results_dir+'/rew_vs_pred_rew'
self.dump_plan_path = results_dir+'/dump_plan'
#if folder do not exists, create it
os.makedirs(self.statistics_path, exist_ok=True)
os.makedirs(self.model_path, exist_ok=True)
os.makedirs(self.video_path, exist_ok=True)
os.makedirs(self.rew_vs_pred_rew_path, exist_ok=True)
os.makedirs(self.dump_plan_path, exist_ok=True)
# Create models
self.transition_model = TransitionModel(self.parms.belief_size, self.parms.state_size, self.env.action_size, self.parms.hidden_size, self.parms.embedding_size, self.parms.activation_function).to(device=self.parms.device)
self.observation_model = ObservationModel(self.parms.belief_size, self.parms.state_size, self.parms.embedding_size, self.parms.activation_function).to(device=self.parms.device)
self.reward_model = RewardModel(self.parms.belief_size, self.parms.state_size, self.parms.hidden_size, self.parms.activation_function).to(device=self.parms.device)
self.encoder = Encoder(self.parms.embedding_size,self.parms.activation_function).to(device=self.parms.device)
self.param_list = list(self.transition_model.parameters()) + list(self.observation_model.parameters()) + list(self.reward_model.parameters()) + list(self.encoder.parameters())
self.optimiser = optim.Adam(self.param_list, lr=0 if self.parms.learning_rate_schedule != 0 else self.parms.learning_rate, eps=self.parms.adam_epsilon)
self.planner = MPCPlanner(self.env.action_size, self.parms.planning_horizon, self.parms.optimisation_iters, self.parms.candidates, self.parms.top_candidates, self.transition_model, self.reward_model,self.env.action_range[0], self.env.action_range[1])
global_prior = Normal(torch.zeros(self.parms.batch_size, self.parms.state_size, device=self.parms.device), torch.ones(self.parms.batch_size, self.parms.state_size, device=self.parms.device)) # Global prior N(0, I)
self.free_nats = torch.full((1, ), self.parms.free_nats, dtype=torch.float32, device=self.parms.device) # Allowed deviation in KL divergence
def load_checkpoints(self):
self.metrics = torch.load(self.model_path+'/metrics.pth')
model_path = self.model_path+'/best_model'
os.makedirs(model_path, exist_ok=True)
files = os.listdir(model_path)
if files:
checkpoint = [f for f in files if os.path.isfile(os.path.join(model_path, f))]
model_dicts = torch.load(os.path.join(model_path, checkpoint[0]),map_location=self.parms.device)
self.transition_model.load_state_dict(model_dicts['transition_model'])
self.observation_model.load_state_dict(model_dicts['observation_model'])
self.reward_model.load_state_dict(model_dicts['reward_model'])
self.encoder.load_state_dict(model_dicts['encoder'])
self.optimiser.load_state_dict(model_dicts['optimiser'])
print("Loading models checkpoints!")
else:
print("Checkpoints not found!")
def update_belief_and_act(self, env, belief, posterior_state, action, observation, reward, min_action=-inf, max_action=inf,explore=False):
# Infer belief over current state q(s_t|o≤t,a<t) from the history
encoded_obs = self.encoder(observation).unsqueeze(dim=0).to(device=self.parms.device)
belief, _, _, _, posterior_state, _, _ = self.transition_model(posterior_state, action.unsqueeze(dim=0), belief, encoded_obs) # Action and observation need extra time dimension
belief, posterior_state = belief.squeeze(dim=0), posterior_state.squeeze(dim=0) # Remove time dimension from belief/state
action,pred_next_rew,_,_,_ = self.planner(belief, posterior_state,explore) # Get action from planner(q(s_t|o≤t,a<t), p)
if explore:
action = action + self.parms.action_noise * torch.randn_like(action) # Add exploration noise ε ~ p(ε) to the action
action.clamp_(min=min_action, max=max_action) # Clip action range
next_observation, reward, done = env.step(action.cpu() if isinstance(env, EnvBatcher) else action[0].cpu()) # If single env is istanceted perform single action (get item from list), else perform all actions
return belief, posterior_state, action, next_observation, reward, done,pred_next_rew
def fit_buffer(self,episode):
####
# Fit data taken from buffer
######
# Model fitting
losses = []
tqdm.write("Fitting buffer")
for s in tqdm(range(self.parms.collect_interval)):
# Draw sequence chunks {(o_t, a_t, r_t+1, terminal_t+1)} ~ D uniformly at random from the dataset (including terminal flags)
observations, actions, rewards, nonterminals = self.D.sample(self.parms.batch_size, self.parms.chunk_size) # Transitions start at time t = 0
# Create initial belief and state for time t = 0
init_belief, init_state = torch.zeros(self.parms.batch_size, self.parms.belief_size, device=self.parms.device), torch.zeros(self.parms.batch_size, self.parms.state_size, device=self.parms.device)
encoded_obs = bottle(self.encoder, (observations[1:], ))
# Update belief/state using posterior from previous belief/state, previous action and current observation (over entire sequence at once)
beliefs, prior_states, prior_means, prior_std_devs, posterior_states, posterior_means, posterior_std_devs = self.transition_model(init_state, actions[:-1], init_belief, encoded_obs, nonterminals[:-1])
# Calculate observation likelihood, reward likelihood and KL losses (for t = 0 only for latent overshooting); sum over final dims, average over batch and time (original implementation, though paper seems to miss 1/T scaling?)
# LOSS
observation_loss = F.mse_loss(bottle(self.observation_model, (beliefs, posterior_states)), observations[1:], reduction='none').sum((2, 3, 4)).mean(dim=(0, 1))
kl_loss = torch.max(kl_divergence(Normal(posterior_means, posterior_std_devs), Normal(prior_means, prior_std_devs)).sum(dim=2), self.free_nats).mean(dim=(0, 1))
reward_loss = F.mse_loss(bottle(self.reward_model, (beliefs, posterior_states)), rewards[:-1], reduction='none').mean(dim=(0, 1))
# Update model parameters
self.optimiser.zero_grad()
(observation_loss + reward_loss + kl_loss).backward() # BACKPROPAGATION
nn.utils.clip_grad_norm_(self.param_list, self.parms.grad_clip_norm, norm_type=2)
self.optimiser.step()
# Store (0) observation loss (1) reward loss (2) KL loss
losses.append([observation_loss.item(), reward_loss.item(), kl_loss.item()])#, regularizer_loss.item()])
#save statistics and plot them
losses = tuple(zip(*losses))
self.metrics['observation_loss'].append(losses[0])
self.metrics['reward_loss'].append(losses[1])
self.metrics['kl_loss'].append(losses[2])
lineplot(self.metrics['episodes'][-len(self.metrics['observation_loss']):], self.metrics['observation_loss'], 'observation_loss', self.statistics_path)
lineplot(self.metrics['episodes'][-len(self.metrics['reward_loss']):], self.metrics['reward_loss'], 'reward_loss', self.statistics_path)
lineplot(self.metrics['episodes'][-len(self.metrics['kl_loss']):], self.metrics['kl_loss'], 'kl_loss', self.statistics_path)
def explore_and_collect(self,episode):
tqdm.write("Collect new data:")
reward = 0
# Data collection
with torch.no_grad():
done = False
observation, total_reward = self.env.reset(), 0
belief, posterior_state, action = torch.zeros(1, self.parms.belief_size, device=self.parms.device), torch.zeros(1, self.parms.state_size, device=self.parms.device), torch.zeros(1, self.env.action_size, device=self.parms.device)
t = 0
real_rew = []
predicted_rew = []
total_steps = self.parms.max_episode_length // self.env.action_repeat
explore = True
for t in tqdm(range(total_steps)):
# Here we need to explore
belief, posterior_state, action, next_observation, reward, done, pred_next_rew = self.update_belief_and_act(self.env, belief, posterior_state, action, observation.to(device=self.parms.device), [reward], self.env.action_range[0], self.env.action_range[1], explore=explore)
self.D.append(observation, action.cpu(), reward, done)
real_rew.append(reward)
predicted_rew.append(pred_next_rew.to(device=self.parms.device).item())
total_reward += reward
observation = next_observation
if self.parms.flag_render:
env.render()
if done:
break
# Update and plot train reward metrics
self.metrics['steps'].append( (t * self.env.action_repeat) + self.metrics['steps'][-1])
self.metrics['episodes'].append(episode)
self.metrics['train_rewards'].append(total_reward)
self.metrics['predicted_rewards'].append(np.array(predicted_rew).sum())
lineplot(self.metrics['episodes'][-len(self.metrics['train_rewards']):], self.metrics['train_rewards'], 'train_rewards', self.statistics_path)
double_lineplot(self.metrics['episodes'], self.metrics['train_rewards'], self.metrics['predicted_rewards'], "train_r_vs_pr", self.statistics_path)
def train_models(self):
# from (init_episodes) to (training_episodes + init_episodes)
tqdm.write("Start training.")
for episode in tqdm(range(self.parms.num_init_episodes +1, self.parms.training_episodes) ):
self.fit_buffer(episode)
self.explore_and_collect(episode)
if episode % self.parms.test_interval == 0:
self.test_model(episode)
torch.save(self.metrics, os.path.join(self.model_path, 'metrics.pth'))
torch.save({'transition_model': self.transition_model.state_dict(), 'observation_model': self.observation_model.state_dict(), 'reward_model': self.reward_model.state_dict(), 'encoder': self.encoder.state_dict(), 'optimiser': self.optimiser.state_dict()}, os.path.join(self.model_path, 'models_%d.pth' % episode))
if episode % self.parms.storing_dataset_interval == 0:
self.D.store_dataset(self.parms.dataset_path+'dump_dataset')
return self.metrics
def test_model(self, episode=None): #no explore here
if episode is None:
episode = self.tested_episodes
# Set models to eval mode
self.transition_model.eval()
self.observation_model.eval()
self.reward_model.eval()
self.encoder.eval()
# Initialise parallelised test environments
test_envs = EnvBatcher(ControlSuiteEnv, (self.parms.env_name, self.parms.seed, self.parms.max_episode_length, self.parms.bit_depth), {}, self.parms.test_episodes)
total_steps = self.parms.max_episode_length // test_envs.action_repeat
rewards = np.zeros(self.parms.test_episodes)
real_rew = torch.zeros([total_steps,self.parms.test_episodes])
predicted_rew = torch.zeros([total_steps,self.parms.test_episodes])
with torch.no_grad():
observation, total_rewards, video_frames = test_envs.reset(), np.zeros((self.parms.test_episodes, )), []
belief, posterior_state, action = torch.zeros(self.parms.test_episodes, self.parms.belief_size, device=self.parms.device), torch.zeros(self.parms.test_episodes, self.parms.state_size, device=self.parms.device), torch.zeros(self.parms.test_episodes, self.env.action_size, device=self.parms.device)
tqdm.write("Testing model.")
for t in range(total_steps):
belief, posterior_state, action, next_observation, rewards, done, pred_next_rew = self.update_belief_and_act(test_envs, belief, posterior_state, action, observation.to(device=self.parms.device), list(rewards), self.env.action_range[0], self.env.action_range[1])
total_rewards += rewards.numpy()
real_rew[t] = rewards
predicted_rew[t] = pred_next_rew
observation = self.env.get_original_frame().unsqueeze(dim=0)
video_frames.append(make_grid(torch.cat([observation, self.observation_model(belief, posterior_state).cpu()], dim=3) + 0.5, nrow=5).numpy()) # Decentre
observation = next_observation
if done.sum().item() == self.parms.test_episodes:
break
real_rew = torch.transpose(real_rew, 0, 1)
predicted_rew = torch.transpose(predicted_rew, 0, 1)
#save and plot metrics
self.tested_episodes += 1
self.metrics['test_episodes'].append(episode)
self.metrics['test_rewards'].append(total_rewards.tolist())
lineplot(self.metrics['test_episodes'], self.metrics['test_rewards'], 'test_rewards', self.statistics_path)
write_video(video_frames, 'test_episode_%s' % str(episode), self.video_path) # Lossy compression
# Set models to train mode
self.transition_model.train()
self.observation_model.train()
self.reward_model.train()
self.encoder.train()
# Close test environments
test_envs.close()
return self.metrics
def dump_plan_video(self, step_before_plan=120):
#number of steps before to start to collect frames to dump
step_before_plan = min(step_before_plan, (self.parms.max_episode_length // self.env.action_repeat))
# Set models to eval mode
self.transition_model.eval()
self.observation_model.eval()
self.reward_model.eval()
self.encoder.eval()
video_frames = []
reward = 0
with torch.no_grad():
observation = self.env.reset()
belief, posterior_state, action = torch.zeros(1, self.parms.belief_size, device=self.parms.device), torch.zeros(1, self.parms.state_size, device=self.parms.device), torch.zeros(1, self.env.action_size, device=self.parms.device)
tqdm.write("Executing episode.")
for t in range(step_before_plan): #floor division
belief, posterior_state, action, next_observation, reward, done, _ = self.update_belief_and_act(self.env, belief, posterior_state, action, observation.to(device=self.parms.device), [reward], self.env.action_range[0], self.env.action_range[1])
observation = next_observation
video_frames.append(make_grid(torch.cat([observation.cpu(), self.observation_model(belief, posterior_state).to(device=self.parms.device).cpu()], dim=3) + 0.5, nrow=5).numpy()) # Decentre
if done:
break
self.create_and_dump_plan(self.env, belief, posterior_state, action, observation.to(device=self.parms.device), [reward], self.env.action_range[0], self.env.action_range[1])
# Set models to train mode
self.transition_model.train()
self.observation_model.train()
self.reward_model.train()
self.encoder.train()
# Close test environments
self.env.close()
def create_and_dump_plan(self, env, belief, posterior_state, action, observation, reward, min_action=-inf, max_action=inf):
tqdm.write("Dumping plan")
video_frames = []
encoded_obs = self.encoder(observation).unsqueeze(dim=0)
belief, _, _, _, posterior_state, _, _ = self.transition_model(posterior_state, action.unsqueeze(dim=0), belief, encoded_obs)
belief, posterior_state = belief.squeeze(dim=0), posterior_state.squeeze(dim=0) # Remove time dimension from belief/state
next_action,_, beliefs, states, plan = self.planner(belief, posterior_state,False) # Get action from planner(q(s_t|o≤t,a<t), p)
predicted_frames = self.observation_model(beliefs, states).to(device=self.parms.device)
for i in range(self.parms.planning_horizon):
plan[i].clamp_(min=env.action_range[0], max=self.env.action_range[1]) # Clip action range
next_observation, reward, done = env.step(plan[i].cpu())
next_observation = next_observation.squeeze(dim=0)
video_frames.append(make_grid(torch.cat([next_observation, predicted_frames[i]], dim=1) + 0.5, nrow=2).numpy()) # Decentre
write_video(video_frames, 'dump_plan', self.dump_plan_path, dump_frame=True)
class Dreamer(Agent):
# The agent has its own replay buffer, update, act
def __init__(self, args):
"""
All paras are passed by args
:param args: a dict that includes parameters
"""
super().__init__()
self.args = args
# Initialise model parameters randomly
self.transition_model = TransitionModel(
args.belief_size, args.state_size, args.action_size,
args.hidden_size, args.embedding_size,
args.dense_act).to(device=args.device)
self.observation_model = ObservationModel(
args.symbolic,
args.observation_size,
args.belief_size,
args.state_size,
args.embedding_size,
activation_function=(args.dense_act if args.symbolic else
args.cnn_act)).to(device=args.device)
self.reward_model = RewardModel(args.belief_size, args.state_size,
args.hidden_size,
args.dense_act).to(device=args.device)
self.encoder = Encoder(args.symbolic, args.observation_size,
args.embedding_size,
args.cnn_act).to(device=args.device)
self.actor_model = ActorModel(
args.action_size,
args.belief_size,
args.state_size,
args.hidden_size,
activation_function=args.dense_act,
fix_speed=args.fix_speed,
throttle_base=args.throttle_base).to(device=args.device)
self.value_model = ValueModel(args.belief_size, args.state_size,
args.hidden_size,
args.dense_act).to(device=args.device)
self.value_model2 = ValueModel(args.belief_size, args.state_size,
args.hidden_size,
args.dense_act).to(device=args.device)
self.pcont_model = PCONTModel(args.belief_size, args.state_size,
args.hidden_size,
args.dense_act).to(device=args.device)
self.target_value_model = deepcopy(self.value_model)
self.target_value_model2 = deepcopy(self.value_model2)
for p in self.target_value_model.parameters():
p.requires_grad = False
for p in self.target_value_model2.parameters():
p.requires_grad = False
# setup the paras to update
self.world_param = list(self.transition_model.parameters())\
+ list(self.observation_model.parameters())\
+ list(self.reward_model.parameters())\
+ list(self.encoder.parameters())
if args.pcont:
self.world_param += list(self.pcont_model.parameters())
# setup optimizer
self.world_optimizer = optim.Adam(self.world_param, lr=args.world_lr)
self.actor_optimizer = optim.Adam(self.actor_model.parameters(),
lr=args.actor_lr)
self.value_optimizer = optim.Adam(list(self.value_model.parameters()) +
list(self.value_model2.parameters()),
lr=args.value_lr)
# setup the free_nat to
self.free_nats = torch.full(
(1, ), args.free_nats, dtype=torch.float32,
device=args.device) # Allowed deviation in KL divergence
# TODO: change it to the new replay buffer, in buffer.py
self.D = ExperienceReplay(args.experience_size, args.symbolic,
args.observation_size, args.action_size,
args.bit_depth, args.device)
if self.args.auto_temp:
# setup for learning of alpha term (temp of the entropy term)
self.log_temp = torch.zeros(1,
requires_grad=True,
device=args.device)
self.target_entropy = -np.prod(
args.action_size if not args.fix_speed else self.args.
action_size - 1).item() # heuristic value from SAC paper
self.temp_optimizer = optim.Adam(
[self.log_temp], lr=args.value_lr) # use the same value_lr
# TODO: print out the param used in Dreamer
# var_counts = tuple(count_vars(module) for module in [self., self.ac.q1, self.ac.q2])
# print('\nNumber of parameters: \t pi: %d, \t q1: %d, \t q2: %d\n' % var_counts)
# def process_im(self, image, image_size=None, rgb=None):
# # Resize, put channel first, convert it to a tensor, centre it to [-0.5, 0.5] and add batch dimenstion.
#
# def preprocess_observation_(observation, bit_depth):
# # Preprocesses an observation inplace (from float32 Tensor [0, 255] to [-0.5, 0.5])
# observation.div_(2 ** (8 - bit_depth)).floor_().div_(2 ** bit_depth).sub_(
# 0.5) # Quantise to given bit depth and centre
# observation.add_(torch.rand_like(observation).div_(
# 2 ** bit_depth)) # Dequantise (to approx. match likelihood of PDF of continuous images vs. PMF of discrete images)
#
# image = image[40:, :, :] # clip the above 40 rows
# image = torch.tensor(cv2.resize(image, (40, 40), interpolation=cv2.INTER_LINEAR).transpose(2, 0, 1),
# dtype=torch.float32) # Resize and put channel first
#
# preprocess_observation_(image, self.args.bit_depth)
# return image.unsqueeze(dim=0)
def process_im(self, images, image_size=None, rgb=None):
images = cv2.resize(images, (40, 40))
images = np.dot(images, [0.299, 0.587, 0.114])
obs = torch.tensor(images,
dtype=torch.float32).div_(255.).sub_(0.5).unsqueeze(
dim=0) # shape [1, 40, 40], range:[-0.5,0.5]
return obs.unsqueeze(dim=0) # add batch dimension
def append_buffer(self, new_traj):
# append new collected trajectory, not implement the data augmentation
# shape of new_traj: [(o, a, r, d) * steps]
for state in new_traj:
observation, action, reward, done = state
self.D.append(observation, action.cpu(), reward, done)
def _compute_loss_world(self, state, data):
# unpackage data
beliefs, prior_states, prior_means, prior_std_devs, posterior_states, posterior_means, posterior_std_devs = state
observations, rewards, nonterminals = data
# observation_loss = F.mse_loss(
# bottle(self.observation_model, (beliefs, posterior_states)),
# observations[1:],
# reduction='none').sum(dim=2 if self.args.symbolic else (2, 3, 4)).mean(dim=(0, 1))
#
# reward_loss = F.mse_loss(
# bottle(self.reward_model, (beliefs, posterior_states)),
# rewards[1:],
# reduction='none').mean(dim=(0,1))
observation_loss = F.mse_loss(
bottle(self.observation_model, (beliefs, posterior_states)),
observations,
reduction='none').sum(
dim=2 if self.args.symbolic else (2, 3, 4)).mean(dim=(0, 1))
reward_loss = F.mse_loss(bottle(self.reward_model,
(beliefs, posterior_states)),
rewards,
reduction='none').mean(dim=(0, 1)) # TODO: 5
# transition loss
kl_loss = torch.max(
kl_divergence(
Independent(Normal(posterior_means, posterior_std_devs), 1),
Independent(Normal(prior_means, prior_std_devs), 1)),
self.free_nats).mean(dim=(0, 1))
# print("check the reward", bottle(pcont_model, (beliefs, posterior_states)).shape, nonterminals[:-1].shape)
if self.args.pcont:
pcont_loss = F.binary_cross_entropy(
bottle(self.pcont_model, (beliefs, posterior_states)),
nonterminals)
# pcont_pred = torch.distributions.Bernoulli(logits=bottle(self.pcont_model, (beliefs, posterior_states)))
# pcont_loss = -pcont_pred.log_prob(nonterminals[1:]).mean(dim=(0, 1))
return observation_loss, self.args.reward_scale * reward_loss, kl_loss, (
self.args.pcont_scale * pcont_loss if self.args.pcont else 0)
def _compute_loss_actor(self,
imag_beliefs,
imag_states,
imag_ac_logps=None):
# reward and value prediction of imagined trajectories
imag_rewards = bottle(self.reward_model, (imag_beliefs, imag_states))
imag_values = bottle(self.value_model, (imag_beliefs, imag_states))
imag_values2 = bottle(self.value_model2, (imag_beliefs, imag_states))
imag_values = torch.min(imag_values, imag_values2)
with torch.no_grad():
if self.args.pcont:
pcont = bottle(self.pcont_model, (imag_beliefs, imag_states))
else:
pcont = self.args.discount * torch.ones_like(imag_rewards)
pcont = pcont.detach()
if imag_ac_logps is not None:
imag_values[
1:] -= self.args.temp * imag_ac_logps # add entropy here
returns = cal_returns(imag_rewards[:-1],
imag_values[:-1],
imag_values[-1],
pcont[:-1],
lambda_=self.args.disclam)
discount = torch.cumprod(
torch.cat([torch.ones_like(pcont[:1]), pcont[:-2]], 0), 0)
discount = discount.detach()
assert list(discount.size()) == list(returns.size())
actor_loss = -torch.mean(discount * returns)
return actor_loss
def _compute_loss_critic(self,
imag_beliefs,
imag_states,
imag_ac_logps=None):
with torch.no_grad():
# calculate the target with the target nn
target_imag_values = bottle(self.target_value_model,
(imag_beliefs, imag_states))
target_imag_values2 = bottle(self.target_value_model2,
(imag_beliefs, imag_states))
target_imag_values = torch.min(target_imag_values,
target_imag_values2)
imag_rewards = bottle(self.reward_model,
(imag_beliefs, imag_states))
if self.args.pcont:
pcont = bottle(self.pcont_model, (imag_beliefs, imag_states))
else:
pcont = self.args.discount * torch.ones_like(imag_rewards)
# print("check pcont", pcont)
if imag_ac_logps is not None:
target_imag_values[1:] -= self.args.temp * imag_ac_logps
returns = cal_returns(imag_rewards[:-1],
target_imag_values[:-1],
target_imag_values[-1],
pcont[:-1],
lambda_=self.args.disclam)
target_return = returns.detach()
value_pred = bottle(self.value_model, (imag_beliefs, imag_states))[:-1]
value_pred2 = bottle(self.value_model2,
(imag_beliefs, imag_states))[:-1]
value_loss = F.mse_loss(value_pred, target_return,
reduction="none").mean(dim=(0, 1))
value_loss2 = F.mse_loss(value_pred2, target_return,
reduction="none").mean(dim=(0, 1))
value_loss += value_loss2
return value_loss
def _latent_imagination(self,
beliefs,
posterior_states,
with_logprob=False):
# Rollout to generate imagined trajectories
chunk_size, batch_size, _ = list(
posterior_states.size()) # flatten the tensor
flatten_size = chunk_size * batch_size
posterior_states = posterior_states.detach().reshape(flatten_size, -1)
beliefs = beliefs.detach().reshape(flatten_size, -1)
imag_beliefs, imag_states, imag_ac_logps = [beliefs
], [posterior_states], []
for i in range(self.args.planning_horizon):
imag_action, imag_ac_logp = self.actor_model(
imag_beliefs[-1].detach(),
imag_states[-1].detach(),
deterministic=False,
with_logprob=with_logprob,
)
imag_action = imag_action.unsqueeze(dim=0) # add time dim
# print(imag_states[-1].shape, imag_action.shape, imag_beliefs[-1].shape)
imag_belief, imag_state, _, _ = self.transition_model(
imag_states[-1], imag_action, imag_beliefs[-1])
imag_beliefs.append(imag_belief.squeeze(dim=0))
imag_states.append(imag_state.squeeze(dim=0))
if with_logprob:
imag_ac_logps.append(imag_ac_logp.squeeze(dim=0))
imag_beliefs = torch.stack(imag_beliefs, dim=0).to(
self.args.device
) # shape [horizon+1, (chuck-1)*batch, belief_size]
imag_states = torch.stack(imag_states, dim=0).to(self.args.device)
if with_logprob:
imag_ac_logps = torch.stack(imag_ac_logps, dim=0).to(
self.args.device) # shape [horizon, (chuck-1)*batch]
return imag_beliefs, imag_states, imag_ac_logps if with_logprob else None
def update_parameters(self, gradient_steps):
loss_info = [] # used to record loss
for s in tqdm(range(gradient_steps)):
# get state and belief of samples
observations, actions, rewards, nonterminals = self.D.sample(
self.args.batch_size, self.args.chunk_size)
# print("check sampled rewrads", rewards)
init_belief = torch.zeros(self.args.batch_size,
self.args.belief_size,
device=self.args.device)
init_state = torch.zeros(self.args.batch_size,
self.args.state_size,
device=self.args.device)
# Update belief/state using posterior from previous belief/state, previous action and current observation (over entire sequence at once)
# beliefs, prior_states, prior_means, prior_std_devs, posterior_states, posterior_means, posterior_std_devs = self.transition_model(
# init_state,
# actions[:-1],
# init_belief,
# bottle(self.encoder, (observations[1:], )),
# nonterminals[:-1])
beliefs, prior_states, prior_means, prior_std_devs, posterior_states, posterior_means, posterior_std_devs = self.transition_model(
init_state, actions, init_belief,
bottle(self.encoder, (observations, )),
nonterminals) # TODO: 4
# update paras of world model
world_model_loss = self._compute_loss_world(
state=(beliefs, prior_states, prior_means, prior_std_devs,
posterior_states, posterior_means, posterior_std_devs),
data=(observations, rewards, nonterminals))
observation_loss, reward_loss, kl_loss, pcont_loss = world_model_loss
self.world_optimizer.zero_grad()
(observation_loss + reward_loss + kl_loss + pcont_loss).backward()
nn.utils.clip_grad_norm_(self.world_param,
self.args.grad_clip_norm,
norm_type=2)
self.world_optimizer.step()
# freeze params to save memory
for p in self.world_param:
p.requires_grad = False
for p in self.value_model.parameters():
p.requires_grad = False
for p in self.value_model2.parameters():
p.requires_gard = False
# latent imagination
imag_beliefs, imag_states, imag_ac_logps = self._latent_imagination(
beliefs, posterior_states, with_logprob=self.args.with_logprob)
# update temp
if self.args.auto_temp:
temp_loss = -(
self.log_temp *
(imag_ac_logps[0] + self.target_entropy).detach()).mean()
self.temp_optimizer.zero_grad()
temp_loss.backward()
self.temp_optimizer.step()
self.args.temp = self.log_temp.exp()
# update actor
actor_loss = self._compute_loss_actor(imag_beliefs,
imag_states,
imag_ac_logps=imag_ac_logps)
self.actor_optimizer.zero_grad()
actor_loss.backward()
nn.utils.clip_grad_norm_(self.actor_model.parameters(),
self.args.grad_clip_norm,
norm_type=2)
self.actor_optimizer.step()
for p in self.world_param:
p.requires_grad = True
for p in self.value_model.parameters():
p.requires_grad = True
for p in self.value_model2.parameters():
p.requires_grad = True
# update critic
imag_beliefs = imag_beliefs.detach()
imag_states = imag_states.detach()
critic_loss = self._compute_loss_critic(
imag_beliefs, imag_states, imag_ac_logps=imag_ac_logps)
self.value_optimizer.zero_grad()
critic_loss.backward()
nn.utils.clip_grad_norm_(self.value_model.parameters(),
self.args.grad_clip_norm,
norm_type=2)
nn.utils.clip_grad_norm_(self.value_model2.parameters(),
self.args.grad_clip_norm,
norm_type=2)
self.value_optimizer.step()
loss_info.append([
observation_loss.item(),
reward_loss.item(),
kl_loss.item(),
pcont_loss.item() if self.args.pcont else 0,
actor_loss.item(),
critic_loss.item()
])
# finally, update target value function every #gradient_steps
with torch.no_grad():
self.target_value_model.load_state_dict(
self.value_model.state_dict())
with torch.no_grad():
self.target_value_model2.load_state_dict(
self.value_model2.state_dict())
return loss_info
def infer_state(self, observation, action, belief=None, state=None):
""" Infer belief over current state q(s_t|o≤t,a<t) from the history,
return updated belief and posterior_state at time t
returned shape: belief/state [belief/state_dim] (remove the time_dim)
"""
# observation is obs.to(device), action.shape=[act_dim] (will add time dim inside this fn), belief.shape
belief, _, _, _, posterior_state, _, _ = self.transition_model(
state, action.unsqueeze(dim=0), belief,
self.encoder(observation).unsqueeze(
dim=0)) # Action and observation need extra time dimension
belief, posterior_state = belief.squeeze(
dim=0), posterior_state.squeeze(
dim=0) # Remove time dimension from belief/state
return belief, posterior_state
def select_action(self, state, deterministic=False):
# get action with the inputs get from fn: infer_state; return a numpy with shape [batch, act_size]
belief, posterior_state = state
action, _ = self.actor_model(belief,
posterior_state,
deterministic=deterministic,
with_logprob=False)
if not deterministic and not self.args.with_logprob:
print("e")
action = Normal(action, self.args.expl_amount).rsample()
# clip the angle
action[:, 0].clamp_(min=self.args.angle_min,
max=self.args.angle_max)
# clip the throttle
if self.args.fix_speed:
action[:, 1] = self.args.throttle_base
else:
action[:, 1].clamp_(min=self.args.throttle_min,
max=self.args.throttle_max)
print("action", action)
# return action.cup().numpy()
return action # this is a Tonsor.cuda
def import_parameters(self, params):
# only import or export the parameters used when local rollout
self.encoder.load_state_dict(params["encoder"])
self.actor_model.load_state_dict(params["policy"])
self.transition_model.load_state_dict(params["transition"])
def export_parameters(self):
""" return the model paras used for local rollout """
params = {
"encoder": self.encoder.cpu().state_dict(),
"policy": self.actor_model.cpu().state_dict(),
"transition": self.transition_model.cpu().state_dict()
}
self.encoder.to(self.args.device)
self.actor_model.to(self.args.device)
self.transition_model.to(self.args.device)
return params