def __init__(self, name, model, lstm_model, obs_shape_n, act_space_n, agent_index, shared_actor, args, local_q_func=False):
# 最终的目标actor
self.shared_actor = shared_actor
self.name = name
self.n = len(obs_shape_n)
self.agent_index = agent_index
self.args = args
obs_ph_n = []
for i in range(self.n):
obs_shape = list(obs_shape_n[i])
obs_shape.append(args.history_length)
obs_ph_n.append(U.BatchInput((obs_shape), name="observation"+str(i)).get())
# Create all the functions necessary to train the model
self.q_train, self.q_update, self.q_debug = q_train(
scope=self.name,
make_obs_ph_n=obs_ph_n,
act_space_n=act_space_n,
q_index=agent_index,
q_func=model,
lstm_model=lstm_model,
optimizer=tf.train.AdamOptimizer(learning_rate=args.lr),
args=self.args,
grad_norm_clipping=0.5,
local_q_func=local_q_func,
num_units=args.num_units,
reuse=False
)
self.replay_buffer = ReplayBuffer(args, obs_shape_n[0], act_space_n[0].n)
self.max_replay_buffer_len = args.batch_size * args.max_episode_len
self.replay_sample_index = None
self.p = None
def __init__(self, name, model, lstm_model, obs_shape_n, act_space_n, agent_index, actor_env, args, local_q_func=False):
self.args = args
self.name = name
self.n = len(obs_shape_n)
self.agent_index = agent_index
obs_ph_n = []
for i in range(self.n):
obs_shape = list(obs_shape_n[i])
obs_shape.append(args.history_length)
obs_ph_n.append(U.BatchInput((obs_shape), name="observation" + str(i)).get())
optimizer = tf.train.AdamOptimizer(learning_rate=self.args.lr)
self.p_train, self.p_update = p_train(
scope=self.name,
p_scope=actor_env,
make_obs_ph_n=obs_ph_n,
act_space_n=act_space_n,
p_index=self.agent_index,
p_func=model,
q_func=model,
lstm_model=lstm_model,
optimizer=optimizer,
grad_norm_clipping=0.5,
local_q_func=local_q_func,
num_units=self.args.num_units,
reuse=True,
use_lstm=False
)
# Create experience buffer
self.replay_buffer = ReplayBuffer(args, obs_shape_n[0], act_space_n[0].n)
self.max_replay_buffer_len = args.batch_size * args.max_episode_len
self.replay_sample_index = None
def __init__(self,
name,
model,
lstm_model,
obs_shape_n,
act_space_n,
agent_index,
args,
local_q_func=False):
self.critic_scope = None
self.name = name
self.n = len(obs_shape_n)
self.agent_index = agent_index
self.args = args
obs_ph_n = []
for i in range(self.n):
obs_shape = list(obs_shape_n[i])
obs_shape.append(args.history_length)
obs_ph_n.append(
U.BatchInput((obs_shape), name="observation" + str(i)).get())
# Create all the functions necessary to train the model
self.obs_ph_n = obs_ph_n
self.act_space_n = act_space_n
self.model = model
self.lstm_model = lstm_model
self.local_q_func = local_q_func
self.act, self.p_update, self.p_debug = p_act(
scope=self.name,
make_obs_ph_n=self.obs_ph_n,
act_space_n=self.act_space_n,
p_index=self.agent_index,
p_func=self.model,
lstm_model=self.lstm_model,
args=self.args,
reuse=False,
num_units=self.args.num_units)
self.optimizer = tf.train.AdamOptimizer(learning_rate=self.args.lr)
self.p_train = None
# Create experience buffer
self.replay_buffer = ReplayBuffer(args, obs_shape_n[0],
act_space_n[0].n)
self.max_replay_buffer_len = args.batch_size * args.max_episode_len
self.replay_sample_index = None
class MADDPGAgentTrainer(AgentTrainer):
def __init__(self,
name,
model,
lstm_model,
obs_shape_n,
act_space_n,
agent_index,
args,
local_q_func=False):
self.args = args
self.name = name
self.n = len(obs_shape_n)
self.agent_index = agent_index
obs_ph_n = []
for i in range(self.n):
obs_shape = list(obs_shape_n[i])
obs_shape.append(args.history_length)
obs_ph_n.append(
U.BatchInput((obs_shape), name="observation" + str(i)).get())
self.local_q_func = local_q_func
self.act, self.p_debug = p_act(scope=self.name,
make_obs_ph_n=obs_ph_n,
act_space_n=act_space_n,
p_index=self.agent_index,
p_func=model,
lstm_model=lstm_model,
num_units=self.args.num_units,
use_lstm=False,
reuse=False)
# Create experience buffer
self.replay_buffer = ReplayBuffer(args, obs_shape_n[0],
act_space_n[0].n)
self.max_replay_buffer_len = args.batch_size * args.max_episode_len
self.replay_sample_index = None
def action(self, obs, batch_size):
return self.act(*(obs + batch_size))
def experience(self, obs, act, rew, done, terminal):
# Store transition in the replay buffer.
self.replay_buffer.add(obs, act, rew, float(done), terminal)
def __init__(self,
name,
model,
lstm_model,
obs_shape_n,
act_space_n,
agent_index,
args,
local_q_func=False):
self.args = args
self.name = name
self.n = len(obs_shape_n)
self.agent_index = agent_index
obs_ph_n = []
for i in range(self.n):
obs_shape = list(obs_shape_n[i])
obs_shape.append(args.history_length)
obs_ph_n.append(
U.BatchInput((obs_shape), name="observation" + str(i)).get())
self.local_q_func = local_q_func
self.act, self.p_debug = p_act(scope=self.name,
make_obs_ph_n=obs_ph_n,
act_space_n=act_space_n,
p_index=self.agent_index,
p_func=model,
lstm_model=lstm_model,
num_units=self.args.num_units,
use_lstm=False,
reuse=False)
# Create experience buffer
self.replay_buffer = ReplayBuffer(args, obs_shape_n[0],
act_space_n[0].n)
self.max_replay_buffer_len = args.batch_size * args.max_episode_len
self.replay_sample_index = None
class MADDPGAgentTrainer(AgentTrainer):
def __init__(self,
name,
model,
lstm_model,
obs_shape_n,
act_space_n,
agent_index,
args,
local_q_func=False):
self.critic_scope = None
self.name = name
self.n = len(obs_shape_n)
self.agent_index = agent_index
self.args = args
obs_ph_n = []
for i in range(self.n):
obs_shape = list(obs_shape_n[i])
obs_shape.append(args.history_length)
obs_ph_n.append(
U.BatchInput((obs_shape), name="observation" + str(i)).get())
# Create all the functions necessary to train the model
self.obs_ph_n = obs_ph_n
self.act_space_n = act_space_n
self.model = model
self.lstm_model = lstm_model
self.local_q_func = local_q_func
self.act, self.p_update, self.p_debug = p_act(
scope=self.name,
make_obs_ph_n=self.obs_ph_n,
act_space_n=self.act_space_n,
p_index=self.agent_index,
p_func=self.model,
lstm_model=self.lstm_model,
args=self.args,
reuse=False,
num_units=self.args.num_units)
self.optimizer = tf.train.AdamOptimizer(learning_rate=self.args.lr)
self.p_train = None
# Create experience buffer
self.replay_buffer = ReplayBuffer(args, obs_shape_n[0],
act_space_n[0].n)
self.max_replay_buffer_len = args.batch_size * args.max_episode_len
self.replay_sample_index = None
def add_p(self, p_scope):
self.p_train = p_train(scope=self.name,
make_obs_ph_n=self.obs_ph_n,
act_space_n=self.act_space_n,
p_scope=p_scope,
p_index=self.agent_index,
p_func=self.model,
q_func=self.model,
lstm_model=self.lstm_model,
optimizer=self.optimizer,
args=self.args,
grad_norm_clipping=0.5,
local_q_func=self.local_q_func,
num_units=self.args.num_units,
reuse=True)
def change_p(self, p):
self.p_train = p
def action(self, obs, batch_size):
return self.act(*(obs + batch_size))
def experience(self, obs, act, rew, done, terminal):
# Store transition in the replay buffer.
self.replay_buffer.add(obs, act, rew, float(done), terminal)
def preupdate(self):
self.replay_sample_index = None
def update(self, agents, t):
# 这个就是训练actor的
if len(self.replay_buffer) < self.max_replay_buffer_len:
return
if not t % 100 == 0: # only update every 100 steps
return
print("HHHH")
# collect replay sample from all agents
obs_n = [
] # 长度为n的list，list每隔元素为[batch_size, state_size, history_length]
obs_next_n = [] # list的每个元素是[bath_size, action_dim]
act_n = []
for i in range(self.n):
obs, act, rew, obs_next, done = agents[i].replay_buffer.sample()
obs_n.append(obs)
obs_next_n.append(obs_next)
act_n.append(act)
# train p network
p_loss = self.p_train(*(obs_n + act_n + [self.args.batch_size]))
self.p_update()
# print("step: ", t, "p_loss: ", p_loss)
return [p_loss]
class MADDPGAgentTrainer(AgentTrainer):
def __init__(self,
name,
model,
lstm_model,
obs_shape_n,
act_space_n,
agent_index,
actors,
args,
local_q_func=False):
self.actors = actors
self.name = name
self.n = len(obs_shape_n)
self.agent_index = agent_index
self.args = args
obs_ph_n = []
for i in range(self.n):
obs_shape = list(obs_shape_n[i])
obs_shape.append(args.history_length)
obs_ph_n.append(
U.BatchInput((obs_shape), name="observation" + str(i)).get())
# Create all the functions necessary to train the model
self.q_train, self.q_update, self.q_debug = q_train(
scope=self.name,
make_obs_ph_n=obs_ph_n,
act_space_n=act_space_n,
q_index=agent_index,
q_func=model,
lstm_model=lstm_model,
optimizer=tf.train.AdamOptimizer(learning_rate=args.lr),
args=self.args,
grad_norm_clipping=0.5,
local_q_func=local_q_func,
num_units=args.num_units,
reuse=False,
use_lstm=False)
self.replay_buffer = ReplayBuffer(args, obs_shape_n[0],
act_space_n[0].n)
self.max_replay_buffer_len = args.batch_size * args.max_episode_len
self.replay_sample_index = None
def experience(self, obs, act, rew, done, terminal):
# Store transition in the replay buffer.
self.replay_buffer.add(obs, act, rew, float(done), terminal)
def preupdate(self):
self.replay_sample_index = None
def update(self, agents, t):
# 训练critic
if len(
self.replay_buffer
) < self.max_replay_buffer_len: # replay buffer is not large enough
return
if not t % 100 == 0: # only update every 100 steps
return
# print("HHHH")
# collect replay sample from all agents
obs_n = [
] # 长度为n的list，list每隔元素为[batch_size, state_size, history_length]
obs_next_n = [] # list的每个元素是[bath_size, action_dim]
act_n = []
for i in range(self.n):
obs, act, rew, obs_next, done = agents[i].replay_buffer.sample()
obs_n.append(obs)
obs_next_n.append(obs_next)
act_n.append(act)
obs, act, rew, obs_next, done = self.replay_buffer.sample()
# train q network
num_sample = 1
target_q = 0.0
for i in range(num_sample):
target_act_next_n = [
self.actors[i].p_debug['target_act'](*([obs_next_n[j]] +
[self.args.batch_size]))
for j in range(self.n)
]
target_q_next = self.q_debug['target_q_values'](
*(obs_next_n + target_act_next_n + [self.args.batch_size]))
target_q += rew + self.args.gamma * (1.0 - done) * target_q_next
target_q /= num_sample
q_loss = self.q_train(*(obs_n + act_n + [target_q] +
[self.args.batch_size]))
# train p network
self.q_update()
# print("step: ", t, "q_loss: ", q_loss)
return [q_loss, np.mean(target_q), np.mean(rew), np.std(target_q)]