class CommNetAgents:
def __init__(self, args):
self.n_actions = args.n_actions
self.n_agents = args.n_agents
self.state_shape = args.state_shape
self.obs_shape = args.obs_shape
self.policy = COMA(args)
self.args = args
# 根据weights得到概率，然后再根据epsilon选动作
def choose_action(self, weights, avail_actions, epsilon, evaluate=False):
weights = weights.unsqueeze(0)
avail_actions = torch.tensor(avail_actions,
dtype=torch.float32).unsqueeze(0)
action_num = avail_actions.sum(dim=1, keepdim=True).float().repeat(
1, avail_actions.shape[-1]) # 可以选择的动作的个数
# 先将Actor网络的输出通过softmax转换成概率分布
prob = torch.nn.functional.softmax(weights, dim=-1)
# 在训练的时候给概率分布添加噪音
prob = ((1 - epsilon) * prob +
torch.ones_like(prob) * epsilon / action_num)
prob[avail_actions == 0] = 0.0 # 不能执行的动作概率为0
"""
不能执行的动作概率为0之后，prob中的概率和不为1，这里不需要进行正则化，因为torch.distributions.Categorical
会将其进行正则化。要注意在训练的过程中没有用到Categorical，所以训练时取执行的动作对应的概率需要再正则化。
"""
if epsilon == 0 and evaluate:
# 测试时直接选最大的
action = torch.argmax(prob)
else:
action = Categorical(prob).sample().long()
return action
def get_action_weights(self, obs, last_action):
obs = torch.tensor(obs, dtype=torch.float32)
last_action = torch.tensor(last_action, dtype=torch.float32)
inputs = list()
inputs.append(obs)
# 给obs添加上一个动作、agent编号
if self.args.last_action:
inputs.append(last_action)
if self.args.reuse_network:
inputs.append(torch.eye(self.args.n_agents))
inputs = torch.cat([x for x in inputs], dim=1)
if self.args.cuda:
inputs = inputs.cuda()
self.policy.eval_hidden = self.policy.eval_hidden.cuda()
weights, self.policy.eval_hidden = self.policy.eval_rnn(
inputs, self.policy.eval_hidden)
weights = weights.reshape(self.args.n_agents, self.args.n_actions)
return weights.cpu()
def _get_max_episode_len(self, batch):
terminated = batch['terminated']
episode_num = terminated.shape[0]
max_episode_len = 0
for episode_idx in range(episode_num):
for transition_idx in range(self.args.episode_limit):
if terminated[episode_idx, transition_idx, 0] == 1:
if transition_idx + 1 >= max_episode_len:
max_episode_len = transition_idx + 1
break
return max_episode_len
def train(self,
batch,
train_step,
epsilon=None): # coma在训练时也需要epsilon计算动作的执行概率
# 每次学习时，各个episode的长度不一样，因此取其中最长的episode作为所有episode的长度
max_episode_len = self._get_max_episode_len(batch)
for key in batch.keys():
batch[key] = batch[key][:, :max_episode_len]
self.policy.learn(batch, max_episode_len, train_step, epsilon)
if train_step > 0 and train_step % self.args.save_cycle == 0:
self.policy.save_model(train_step)
class Agents:
def __init__(self, args):
self.n_actions = args.n_actions
self.n_agents = args.n_agents
self.state_shape = args.state_shape
self.obs_shape = args.obs_shape
if args.alg == 'vdn':
self.policy = VDN(args)
elif args.alg == 'qmix':
self.policy = QMIX(args)
else:
self.policy = COMA(args)
self.args = args
def choose_action(self,
obs,
last_action,
agent_num,
avail_actions,
epsilon,
evaluate=False):
inputs = obs.copy()
avail_actions_ind = np.nonzero(avail_actions)[0] # 可执行动作对应的index
# 传入的agent_num是一个整数，代表第几个agent，现在要把他变成一个onehot向量
agent_id = np.zeros(self.n_agents)
agent_id[agent_num] = 1.
if self.args.last_action:
inputs = np.hstack((inputs, last_action)) # obs是数组，不能append
if self.args.reuse_network:
inputs = np.hstack((inputs, agent_id))
hidden_state = self.policy.eval_hidden[:, agent_num, :]
# 转化成Tensor,inputs的维度是(42,)，要转化成(1,42)
inputs = torch.tensor(inputs, dtype=torch.float32).unsqueeze(0)
avail_actions = torch.tensor(avail_actions,
dtype=torch.float32).unsqueeze(0)
q_value, self.policy.eval_hidden[:,
agent_num, :] = self.policy.eval_rnn.forward(
inputs, hidden_state)
if self.args.alg == 'coma':
action = self._choose_coma_action(q_value, avail_actions, epsilon,
evaluate)
else:
q_value[avail_actions == 0.0] = -float(
"inf") # 传入的avail_actions参数是一个array
if np.random.uniform() < epsilon:
action = np.random.choice(avail_actions_ind) # action是一个整数
else:
action = torch.argmax(q_value)
return action
def _choose_coma_action(self,
inputs,
avail_actions,
epsilon,
evaluate=False): # inputs是所有动作的q值
action_num = avail_actions.sum(dim=1, keepdim=True).float().repeat(
1, avail_actions.shape[-1]) # 可以选择的动作的个数
# 先将Actor网络的输出通过softmax转换成概率分布
prob = torch.nn.functional.softmax(inputs, dim=-1)
# 在训练的时候给概率分布添加噪音
prob = ((1 - epsilon) * prob +
torch.ones_like(prob) * epsilon / action_num)
prob[avail_actions == 0] = 0.0 # 不能执行的动作概率为0
"""
不能执行的动作概率为0之后，prob中的概率和不为1，这里不需要进行正则化，因为torch.distributions.Categorical
会将其进行正则化。要注意在训练的过程中没有用到Categorical，所以训练时取执行的动作对应的概率需要再正则化。
"""
if epsilon == 0 and evaluate:
# 测试时直接选最大的
action = torch.argmax(prob)
else:
action = Categorical(prob).sample().long()
return action
def _get_max_episode_len(self, batch):
terminated = batch['terminated']
episode_num = terminated.shape[0]
max_episode_len = 0
for episode_idx in range(episode_num):
for transition_idx in range(self.args.episode_limit):
if terminated[episode_idx, transition_idx, 0] == 1:
if transition_idx + 1 >= max_episode_len:
max_episode_len = transition_idx + 1
break
return max_episode_len
def train(self,
batch,
train_step,
epsilon=None): # coma在训练时也需要epsilon计算动作的执行概率
# 每次学习时，各个episode的长度不一样，因此取其中最长的episode作为所有episode的长度
max_episode_len = self._get_max_episode_len(batch)
for key in batch.keys():
batch[key] = batch[key][:, :max_episode_len]
self.policy.learn(batch, max_episode_len, train_step, epsilon)
if train_step > 0 and train_step % self.args.save_cycle == 0:
self.policy.save_model(train_step)
