def sample(self,
num_steps=1,
fast_lr=0.5,
gamma=0.95,
gae_lambda=1.0,
device='cpu'):
"""
基于初始策略采样训练轨迹，并基于REINFORCE损失调整策略
内循环中，梯度更新使用`first_order=True`，因其仅用于采样轨迹，而不是优化
Sample the training trajectories with the initial policy and adapt the
policy to the task, based on the REINFORCE loss computed on the
training trajectories. The gradient update in the fast adaptation uses
`first_order=True` no matter if the second order version of MAML is
applied since this is only used for sampling trajectories, and not
for optimization.
"""
"""
训练阶段：
采样训练轨迹数据 train_episodes，计算loss，更新原有网络参数
采样验证轨迹数据 valid_episodes
MAML 内部循环更新num_steps次 inner loop / fast adaptation
"""
# 此处参数设置为 None，调用 OrderDict() 参数
"""
******************************************************************
"""
# params = Non
# params_show_multi_task_sampler = self.policy.state_dict()
for step in range(num_steps):
# 获取该batch中所有的轨迹数据，将数据保存至 train_episodes
train_episodes = self.create_episodes(gamma=gamma,
gae_lambda=gae_lambda,
device=device)
"""
计算 reinforce loss， 更新网络参数 params
"""
loss_per_task = reinforce_loss(self.policy, train_episodes)
# 保存平均 reward
avg_reward = train_episodes.rewards.mean()
last_reward = train_episodes.rewards[-1].mean()
# lr = 1e-3
# self.policy.train()
# optimizer = optim.Adam(self.policy.parameters(), lr)
# grad_step(loss, optimizer)
return loss_per_task, avg_reward, last_reward, train_episodes
async def surrogate_loss(self,
train_futures,
valid_futures,
old_pi=None):
first_order = (old_pi is not None) or self.first_order
# 暂停协程函数，等待协程函数 adapt() 运行结束并更新 self.original_policy
# 要先在此处暂停函数
train_loss = await self.adapt(train_futures,
first_order=first_order)
# valid_loss = reinforce_loss(self.original_policy,
# await futures)
valid_episodes = await valid_futures
"""
"""
valid_loss = reinforce_loss(self.original_policy,
valid_episodes)
"""
要等到上面的 train_futures 进行完之后，再往下进行
每一个 train_futures 要对应一个 valid_futures，每一对是并行分开运行的
future 的数量就是 每一个 batch 中 tasks 的数量
"""
# with torch.set_grad_enabled(old_pi is None):
# # 暂停协程函数，等待协程对象 valid_futures 运行结束并输出返回值
# valid_episodes = await valid_futures
# pi = self.policy(valid_episodes.observations, params=params)
#
# if old_pi is None:
# old_pi = detach_distribution(pi)
#
# log_ratio = (pi.log_prob(valid_episodes.actions)
# - old_pi.log_prob(valid_episodes.actions))
# ratio = torch.exp(log_ratio)
#
# losses = -weighted_mean(ratio * valid_episodes.advantages,
# lengths=valid_episodes.lengths)
# kls = weighted_mean(kl_divergence(pi, old_pi),
# lengths=valid_episodes.lengths)
return train_loss, valid_loss
async def adapt(self, train_futures, first_order=None):
# if first_order is None:
# first_order = self.first_order
# Loop over the number of steps of adaptation 循环调整的步数
params = None
# await后面调用future对象，中断当前程序直到得到 futures 的返回值
# 等待 futures 计算完成，再进行计算 reinforce_loss
params_show_maml_trpo = self.policy.state_dict()
for train_future in train_futures:
"""
"""
train_loss = reinforce_loss(self.original_policy,
await train_future)
lr = 1e-3
self.original_policy.train()
optimizer = optim.Adam(self.original_policy.parameters(), lr)
# Take gradient step:
# 计算梯度 已经
grad_step(train_loss, optimizer)
"""
原来的算法
"""
# inner_loss = reinforce_loss(self.policy,
# await futures)
# # 计算更新后参数，好像不传输到网络中？ self.policy.state_dict()仍然为参数
# params = self.policy.update_params(inner_loss,
# params=params,
# step_size=self.fast_lr,
# first_order=first_order)
# params_show_maml_trpo_test = self.policy.state_dict()
return train_loss
def sample(self,
index,
num_steps=1,
fast_lr=0.5,
gamma=0.95,
gae_lambda=1.0,
device='cpu'):
"""
基于初始策略采样训练轨迹，并基于REINFORCE损失调整策略
内循环中，梯度更新使用`first_order=True`，因其仅用于采样轨迹，而不是优化
Sample the training trajectories with the initial policy and adapt the
policy to the task, based on the REINFORCE loss computed on the
training trajectories. The gradient update in the fast adaptation uses
`first_order=True` no matter if the second order version of MAML is
applied since this is only used for sampling trajectories, and not
for optimization.
"""
"""
训练阶段：
采样训练轨迹数据 train_episodes，计算loss，更新原有网络参数
采样验证轨迹数据 valid_episodes
MAML 内部循环更新num_steps次 inner loop / fast adaptation
"""
# 此处参数设置为 None，调用 OrderDict() 参数
"""
******************************************************************
"""
# params = None
# params_show_multi_task_sampler = self.policy.state_dict()
for step in range(num_steps):
# 获取该batch中所有的轨迹数据，将数据保存至 train_episodes
train_episodes = self.create_episodes(gamma=gamma,
gae_lambda=gae_lambda,
device=device)
train_episodes.log('_enqueueAt', datetime.now(timezone.utc))
# QKFIX: Deep copy the episodes before sending them to their
# respective queues, to avoid a race condition. This issue would
# cause the policy pi = policy(observations) to be miscomputed for
# some timesteps, which in turns makes the loss explode.
self.train_queue.put((index, step, deepcopy(train_episodes)))
"""
计算 reinforce loss， 更新网络参数 params
"""
# 多线程程序中，安全使用可变对象
# with + lock：保证每次只有一个线程执行下面代码块
# with 语句会在这个代码块执行前自动获取锁，在执行结束后自动释放锁
with self.policy_lock:
"""
******************************************************************
"""
loss = reinforce_loss(self.policy, train_episodes)
lr = 1e-3
self.policy.train()
optimizer = optim.Adam(self.policy.parameters(), lr)
# Take gradient step:
# 计算梯度 已经
grad_step(loss, optimizer)
# params = self.policy.update_params(loss,
# params=params,
# step_size=fast_lr,
# first_order=True)
"""
******************************************************************
"""
# params_show_multi_task_sampler_test = self.policy.state_dict()
# Sample the validation trajectories with the adapted policy
valid_episodes = self.create_episodes(gamma=gamma,
gae_lambda=gae_lambda,
device=device)
valid_episodes.log('_enqueueAt', datetime.now(timezone.utc))
self.valid_queue.put((index, None, deepcopy(valid_episodes)))