def __init__(self,
agent_init_params,
sa_size,
gamma=0.95,
tau=0.01,
attend_tau=0.002,
pi_lr=0.01,
q_lr=0.01,
reward_scale=10.,
pol_hidden_dim=128,
critic_hidden_dim=128,
attend_heads=4,
**kwargs):
"""
Inputs:
agent_init_params (list of dict): List of dicts with parameters to
initialize each agent
num_in_pol (int): Input dimensions to policy
num_out_pol (int): Output dimensions to policy
sa_size (list of (int, int)): Size of state and action space for
each agent
gamma (float): Discount factor
tau (float): Target update rate
pi_lr (float): Learning rate for policy
q_lr (float): Learning rate for critic
reward_scale (float): Scaling for reward (has effect of optimal
policy entropy)
hidden_dim (int): Number of hidden dimensions for networks
"""
self.nagents = len(sa_size)
self.agents = [
AttentionAgent(lr=pi_lr, hidden_dim=pol_hidden_dim, **params)
for params in agent_init_params
]
self.critic = AttentionCritic(sa_size,
hidden_dim=critic_hidden_dim,
attend_heads=attend_heads)
self.target_critic = AttentionCritic(sa_size,
hidden_dim=critic_hidden_dim,
attend_heads=attend_heads)
hard_update(self.target_critic, self.critic)
self.critic_optimizer = Adam(self.critic.q_parameters(),
lr=q_lr,
weight_decay=1e-3)
self.agent_init_params = agent_init_params
self.gamma = gamma
self.tau = tau
self.attend_tau = attend_tau
self.pi_lr = pi_lr
self.q_lr = q_lr
self.reward_scale = reward_scale
self.pol_dev = 'cpu' # device for policies
self.critic_dev = 'cpu' # device for critics
self.trgt_pol_dev = 'cpu' # device for target policies
self.trgt_critic_dev = 'cpu' # device for target critics
self.niter = 0
def __init__(self, algo_config: List[Tuple[int, int]],
gamma=0.95, tau=0.01, pi_lr=0.01, q_lr=0.01,
reward_scale=10.,
pol_hidden_dim=128,
critic_hidden_dim=128, attend_heads=4,
**kwargs):
"""
Inputs:
algo_config (List[Tuple[int, int]]): Agent types which will exist in this environment
Ex. [(20, 8), (20, 2)]
gamma (float): Discount factor
tau (float): Target update rate
pi_lr (float): Learning rate for policy
q_lr (float): Learning rate for critic
reward_scale (float): Scaling for reward (has effect of optimal
policy entropy)
hidden_dim (int): Number of hidden dimensions for networks
"""
print(algo_config)
# Dictionary which maps agent type to its topology
self.agents = [AttentionAgent(sdim, adim, lr=pi_lr, hidden_dim=pol_hidden_dim) for sdim, adim in algo_config]
self.critic = AttentionCritic(algo_config, hidden_dim=critic_hidden_dim, attend_heads=attend_heads)
self.target_critic = AttentionCritic(algo_config, hidden_dim=critic_hidden_dim, attend_heads=attend_heads)
hard_update(self.target_critic, self.critic)
self.critic_optimizer = Adam(self.critic.parameters(), lr=q_lr, weight_decay=1e-3)
self.gamma = gamma
self.tau = tau
self.pi_lr = pi_lr
self.q_lr = q_lr
self.reward_scale = reward_scale
self.pol_dev = 'cpu' # device for policies
self.critic_dev = 'cpu' # device for critics
self.trgt_pol_dev = 'cpu' # device for target policies
self.trgt_critic_dev = 'cpu' # device for target critics
self.niter = 0
self.init_dict = {'gamma': gamma, 'tau': tau,
'pi_lr': pi_lr, 'q_lr': q_lr,
'reward_scale': reward_scale,
'pol_hidden_dim': pol_hidden_dim,
'critic_hidden_dim': critic_hidden_dim,
'attend_heads': attend_heads,
'algo_config': algo_config}
def __init__(self,
agent_init_params,
sa_size,
gamma=0.95,
tau=0.01,
pi_lr=0.01,
q_lr=0.01,
reward_scale=10.,
pol_hidden_dim=128,
critic_hidden_dim=128,
attend_heads=4,
**kwargs):
"""
Inputs:
agent_init_params (list of dict): List of dicts with parameters to initialize each agent
num_in_pol (int): Input dimensions to policy
num_out_pol (int): Output dimensions to policy
sa_size (list of (int, int)): Size of state and action space for each agent
"""
self.nagents = len(sa_size)
self.agents = [
AttentionAgent(lr=pi_lr, hidden_dim=pol_hidden_dim, **params)
for params in agent_init_params
]
self.critic = AttentionCritic(sa_size,
hidden_dim=critic_hidden_dim,
attend_heads=attend_heads)
self.target_critic = AttentionCritic(sa_size,
hidden_dim=critic_hidden_dim,
attend_heads=attend_heads)
hard_update(self.target_critic, self.critic)
self.critic_optimizer = Adam(self.critic.parameters(),
lr=q_lr,
weight_decay=1e-3)
self.agent_init_params = agent_init_params
self.gamma = gamma
self.tau = tau
self.pi_lr = pi_lr
self.q_lr = q_lr
self.reward_scale = reward_scale
self.niter = 0
def __init__(self,
agent_init_params,
sa_size,
gamma=0.95,
tau=0.01,
pi_lr=0.01,
q_lr=0.01,
reward_scale=10.,
pol_hidden_dim=128,
critic_hidden_dim=128,
attend_heads=4,
l1_reg=0.01,
**kwargs):
"""
Inputs:
agent_init_params (list of dict): List of dicts with parameters to
initialize each agent
num_in_pol (int): Input dimensions to policy
num_out_pol (int): Output dimensions to policy
sa_size (list of (int, int)): Size of state and action space for
each agent
gamma (float): Discount factor
tau (float): Target update rate
pi_lr (float): Learning rate for policy
q_lr (float): Learning rate for critic
reward_scale (float): Scaling for reward (has effect of optimal
policy entropy)
hidden_dim (int): Number of hidden dimensions for networks
"""
self.nagents = len(sa_size)
self.l1_reg = l1_reg
self.agents = [
AttentionAgent(lr=pi_lr, hidden_dim=pol_hidden_dim, **params)
for params in agent_init_params
]
widths = []
hidden_layers = []
running_width = 0
for sdim, adim in sa_size:
totdim = sdim + adim
running_width += totdim
widths.append(running_width)
hidden_layers.append(2)
# def __init__(self, sa_sizes, widths, hidden_layers, selector_width, selector_depth, hidden_dim=32, **kwargs):
self.critic = SelectiveAttentionCritic(sa_size,
widths=widths,
hidden_layers=hidden_layers,
selector_width=2,
selector_depth=2)
self.target_critic = SelectiveAttentionCritic(
sa_size,
widths=widths,
hidden_layers=hidden_layers,
selector_width=2,
selector_depth=2)
hard_update(self.target_critic, self.critic)
self.critic_optimizer = Adam(self.critic.parameters(),
lr=q_lr,
weight_decay=1e-3)
self.agent_init_params = agent_init_params
self.gamma = gamma
self.tau = tau
self.pi_lr = pi_lr
self.q_lr = q_lr
self.reward_scale = reward_scale
self.pol_dev = 'cpu' # device for policies
self.critic_dev = 'cpu' # device for critics
self.trgt_pol_dev = 'cpu' # device for target policies
self.trgt_critic_dev = 'cpu' # device for target critics
self.niter = 0