def learn(self, writer, i_iter): """learn model""" memory, log = self.collector.collect_samples(self.min_batch_size) print( f"Iter: {i_iter}, num steps: {log['num_steps']}, total reward: {log['total_reward']: .4f}, " f"min reward: {log['min_episode_reward']: .4f}, max reward: {log['max_episode_reward']: .4f}, " f"average reward: {log['avg_reward']: .4f}, sample time: {log['sample_time']: .4f}" ) # record reward information with writer.as_default(): tf.summary.scalar("total reward", log['total_reward'], i_iter) tf.summary.scalar("average reward", log['avg_reward'], i_iter) tf.summary.scalar("min reward", log['min_episode_reward'], i_iter) tf.summary.scalar("max reward", log['max_episode_reward'], i_iter) tf.summary.scalar("num steps", log['num_steps'], i_iter) batch = memory.sample() # sample all items in memory batch_state = NDOUBLE(batch.state) batch_action = NDOUBLE(batch.action) batch_reward = NDOUBLE(batch.reward) batch_mask = NDOUBLE(batch.mask) batch_value = self.value_net(batch_state) batch_advantage, batch_return = estimate_advantages( batch_reward, batch_mask, batch_value, self.gamma, self.tau) v_loss, p_loss = vpg_step(self.policy_net, self.value_net, self.optimizer_p, self.optimizer_v, self.vpg_epochs, batch_state, batch_action, batch_return, batch_advantage) return v_loss, p_loss
def learn(self, writer, i_iter): """learn model""" memory, log = self.collector.collect_samples(self.min_batch_size) print( f"Iter: {i_iter}, num steps: {log['num_steps']}, total reward: {log['total_reward']: .4f}, " f"min reward: {log['min_episode_reward']: .4f}, max reward: {log['max_episode_reward']: .4f}, " f"average reward: {log['avg_reward']: .4f}, sample time: {log['sample_time']: .4f}" ) # record reward information with writer.as_default(): tf.summary.scalar("total reward", log["total_reward"], i_iter) tf.summary.scalar("average reward", log["avg_reward"], i_iter) tf.summary.scalar("min reward", log["min_episode_reward"], i_iter) tf.summary.scalar("max reward", log["max_episode_reward"], i_iter) tf.summary.scalar("num steps", log["num_steps"], i_iter) batch = memory.sample() # sample all items in memory batch_state = NDOUBLE(batch.state) batch_action = NDOUBLE(batch.action) batch_reward = NDOUBLE(batch.reward) batch_mask = NDOUBLE(batch.mask) batch_log_prob = NDOUBLE(batch.log_prob)[:, None] batch_value = tf.stop_gradient(self.value_net(batch_state)) batch_advantage, batch_return = estimate_advantages( batch_reward, batch_mask, batch_value, self.gamma, self.tau ) # update by TRPO log_stats = trpo_step( self.policy_net, self.value_net, self.optimizer_v, batch_state, batch_action, batch_log_prob, batch_advantage, batch_return, max_kl=self.max_kl, cg_damping=self.damping, vf_iters=10 ) with writer.as_default(): for k, v in log_stats.items(): tf.summary.scalar(k, v, i_iter) writer.flush() return log_stats
def learn(self, writer, i_iter): """learn model""" memory, log = self.collector.collect_samples(self.min_batch_size) print( f"Iter: {i_iter}, num steps: {log['num_steps']}, total reward: {log['total_reward']: .4f}, " f"min reward: {log['min_episode_reward']: .4f}, max reward: {log['max_episode_reward']: .4f}, " f"average reward: {log['avg_reward']: .4f}, sample time: {log['sample_time']: .4f}" ) # record reward information with writer.as_default(): tf.summary.scalar("total reward", log['total_reward'], i_iter) tf.summary.scalar("average reward", log['avg_reward'], i_iter) tf.summary.scalar("min reward", log['min_episode_reward'], i_iter) tf.summary.scalar("max reward", log['max_episode_reward'], i_iter) tf.summary.scalar("num steps", log['num_steps'], i_iter) batch = memory.sample() # sample all items in memory batch_state = NDOUBLE(batch.state) batch_action = NDOUBLE(batch.action) batch_reward = NDOUBLE(batch.reward) batch_mask = NDOUBLE(batch.mask) batch_log_prob = NDOUBLE(batch.log_prob)[:, None] batch_value = tf.stop_gradient(self.value_net(batch_state)) batch_advantage, batch_return = estimate_advantages( batch_reward, batch_mask, batch_value, self.gamma, self.tau) log_stats = {} if self.ppo_mini_batch_size: batch_size = batch_state.shape[0] mini_batch_num = batch_size // self.ppo_mini_batch_size for e in range(self.ppo_epochs): perm = np.random.permutation(batch_size) for i in range(mini_batch_num): ind = perm[slice( i * self.ppo_mini_batch_size, min(batch_size, (i + 1) * self.ppo_mini_batch_size))] log_stats = ppo_step(self.policy_net, self.value_net, self.optimizer_p, self.optimizer_v, 1, batch_state[ind], batch_action[ind], batch_return[ind], batch_advantage[ind], batch_log_prob[ind], self.clip_epsilon) else: for _ in range(self.ppo_epochs): log_stats = ppo_step(self.policy_net, self.value_net, self.optimizer_p, self.optimizer_v, 1, batch_state, batch_action, batch_return, batch_advantage, batch_log_prob, self.clip_epsilon) with writer.as_default(): tf.summary.histogram("ratio", log_stats["ratio"], i_iter) tf.summary.scalar("policy loss", log_stats["policy_loss"], i_iter) tf.summary.scalar("critic loss", log_stats["critic_loss"], i_iter) writer.flush() return log_stats