def _postprocess_dqn(policy_graph, sample_batch): obs, actions, rewards, new_obs, dones = [ list(x) for x in sample_batch.columns( ["obs", "actions", "rewards", "new_obs", "dones"])] # N-step Q adjustments if policy_graph.config["n_step"] > 1: adjust_nstep( policy_graph.config["n_step"], policy_graph.config["gamma"], obs, actions, rewards, new_obs, dones) batch = SampleBatch({ "obs": obs, "actions": actions, "rewards": rewards, "new_obs": new_obs, "dones": dones, "weights": np.ones_like(rewards)}) # Prioritize on the worker side if batch.count > 0 and policy_graph.config["worker_side_prioritization"]: td_errors = policy_graph.compute_td_error( batch["obs"], batch["actions"], batch["rewards"], batch["new_obs"], batch["dones"], batch["weights"]) new_priorities = ( np.abs(td_errors) + policy_graph.config["prioritized_replay_eps"]) batch.data["weights"] = new_priorities return batch
def _optimize(self): with self.replay_timer: if isinstance(self.replay_buffer, PrioritizedReplayBuffer): (obses_t, actions, rewards, obses_tp1, dones, weights, batch_indexes) = self.replay_buffer.sample( self.train_batch_size, beta=self.prioritized_replay_beta) else: (obses_t, actions, rewards, obses_tp1, dones) = self.replay_buffer.sample(self.train_batch_size) weights = np.ones_like(rewards) batch_indexes = -np.ones_like(rewards) samples = SampleBatch({ "obs": obses_t, "actions": actions, "rewards": rewards, "new_obs": obses_tp1, "dones": dones, "weights": weights, "batch_indexes": batch_indexes }) with self.grad_timer: td_error = self.local_evaluator.compute_apply(samples) new_priorities = (np.abs(td_error) + self.prioritized_replay_eps) if isinstance(self.replay_buffer, PrioritizedReplayBuffer): self.replay_buffer.update_priorities(samples["batch_indexes"], new_priorities) self.grad_timer.push_units_processed(samples.count) self.num_steps_trained += samples.count
def replay(self): with self.replay_timer: if len(self.replay_buffer) < self.replay_starts: return None (obses_t, actions, rewards, obses_tp1, dones, weights, batch_indexes) = self.replay_buffer.sample( self.train_batch_size, beta=self.prioritized_replay_beta) batch = SampleBatch({ "obs": obses_t, "actions": actions, "rewards": rewards, "new_obs": obses_tp1, "dones": dones, "weights": weights, "batch_indexes": batch_indexes}) return batch
def sample(self): obs, actions, rewards, new_obs, dones = [], [], [], [], [] for _ in range(self.config["sample_batch_size"] + self.config["n_step"] - 1): action = self.agent.act(self.state) next_state, reward, done, _ = self.env.step(action) next_state = to_rainbow(next_state) obs.append(self.state.data.cpu().numpy()) actions.append(action) rewards.append(reward) new_obs.append(next_state.data.cpu().numpy()) dones.append(1.0 if done else 0.0) self.state = next_state self.episode_rewards[-1] += reward self.episode_lengths[-1] += 1 if done: self.state = to_rainbow(self.env.reset()) self.agent.reset_noise() self.episode_rewards.append(0.0) self.episode_lengths.append(0.0) self.local_timestep += 1 # N-step Q adjustments if self.config["n_step"] > 1: # Adjust for steps lost from truncation self.local_timestep -= (self.config["n_step"] - 1) adjust_nstep(self.config["n_step"], self.config["gamma"], obs, actions, rewards, new_obs, dones) batch = SampleBatch({ "obs": obs, "actions": actions, "rewards": rewards, "new_obs": new_obs, "dones": dones, "weights": np.ones_like(rewards) }) assert batch.count == self.config["sample_batch_size"] td_errors = self.agent.compute_td_error(batch) batch.data["obs"] = [pack(o) for o in batch["obs"]] batch.data["new_obs"] = [pack(o) for o in batch["new_obs"]] new_priorities = (np.abs(td_errors) + self.config["prioritized_replay_eps"]) batch.data["weights"] = new_priorities return batch
def sample(self): obs, actions, rewards, new_obs, dones = [], [], [], [], [] for _ in range( self.config["sample_batch_size"] + self.config["n_step"] - 1): update_eps = self.exploration.value(self.local_timestep) action = self.act( np.array(self.obs)[None], update_eps=update_eps)[0] obs_tp1, reward, done, _ = self.env.step(action) obs.append(self.obs) actions.append(action) rewards.append(np.sign(reward)) new_obs.append(obs_tp1) dones.append(1.0 if done else 0.0) self.obs = obs_tp1 self.episode_rewards[-1] += reward self.episode_lengths[-1] += 1 if done: self.obs = self.env.reset() self.episode_rewards.append(0.0) self.episode_lengths.append(0.0) self.local_timestep += 1 # N-step Q adjustments if self.config["n_step"] > 1: # Adjust for steps lost from truncation self.local_timestep -= (self.config["n_step"] - 1) adjust_nstep( self.config["n_step"], self.config["gamma"], obs, actions, rewards, new_obs, dones) batch = SampleBatch({ "obs": obs, "actions": actions, "rewards": rewards, "new_obs": new_obs, "dones": dones, "weights": np.ones_like(rewards)}) assert batch.count == self.config["sample_batch_size"] # td_errors = self.agent.compute_td_error(batch) batch.data["obs"] = [pack(o) for o in batch["obs"]] batch.data["new_obs"] = [pack(o) for o in batch["new_obs"]] # new_priorities = ( # np.abs(td_errors) + self.config["prioritized_replay_eps"]) # batch.data["weights"] = new_priorities return batch
def _replay(self): samples = {} with self.replay_timer: for policy_id, replay_buffer in self.replay_buffers.items(): if isinstance(replay_buffer, PrioritizedReplayBuffer): (obses_t, actions, rewards, obses_tp1, dones, weights, batch_indexes) = replay_buffer.sample( self.train_batch_size, beta=self.prioritized_replay_beta) else: (obses_t, actions, rewards, obses_tp1, dones) = replay_buffer.sample(self.train_batch_size) weights = np.ones_like(rewards) batch_indexes = -np.ones_like(rewards) samples[policy_id] = SampleBatch({ "obs": obses_t, "actions": actions, "rewards": rewards, "new_obs": obses_tp1, "dones": dones, "weights": weights, "batch_indexes": batch_indexes }) return MultiAgentBatch(samples, self.train_batch_size)
def sample(self): """sample rollouts from the environment, being called in step in PolicyOptimizer""" observations, rewards, actions, logprobs, dones, values = [], [], [], [], [], [] done = False for step in range(self.config['steps_per_rollout']): value, action, logprob, mean = self.net.forward( to_variable(self.obs[np.newaxis], self.config['cuda'])) action = action.cpu().data.numpy( )[0] if self.config['cuda'] else action.data.numpy()[0] next_obs, reward, done, _ = self.env.step(action) if self.config['cuda']: # torch has an additional dimension for batch size, so we need to select that batch value, logprob, mean = value.data.cpu().numpy()[0], logprob.data.cpu().numpy()[0], \ mean.data.cpu().numpy()[0] else: value, logprob, mean = value.data.numpy()[0], logprob.data.numpy()[0], \ mean.data.numpy()[0] observations.append(self.obs) actions.append(action) rewards.append(reward) logprobs.append(logprob) values.append(value) dones.append(done) self.obs = next_obs if done: # reset the environment self.obs = self.env.reset() if done: last_value = 0.0 else: # bootstrap, we only need the last value to do this value, action, logprob, mean = self.net.forward( to_variable(self.obs[np.newaxis], self.config['cuda'])) if self.config['cuda']: # torch has an additional dimension for batch size, so we need to select that batch value, = value.data.cpu().numpy()[0] else: value, = value.data.numpy()[0] last_value = value # same as ppo_single/model/ppo.py observations = np.asarray(observations) rewards = np.asarray(rewards) logprobs = np.asarray(logprobs) dones = np.asarray(dones) values = np.asarray(values) actions = np.asarray(actions) returns = calculate_returns(rewards, dones, last_value, self.config['gamma']) return SampleBatch({ 'observations': observations, 'rewards': rewards, 'logprobs': logprobs, 'dones': dones, 'values': values, 'actions': actions, 'returns': returns[:-1] })