def __init__(self, agent_init_params, sa_size, gamma=0.95, tau=0.01, attend_tau=0.002, pi_lr=0.01, q_lr=0.01, reward_scale=10., pol_hidden_dim=128, critic_hidden_dim=128, attend_heads=4, **kwargs): """ Inputs: agent_init_params (list of dict): List of dicts with parameters to initialize each agent num_in_pol (int): Input dimensions to policy num_out_pol (int): Output dimensions to policy sa_size (list of (int, int)): Size of state and action space for each agent gamma (float): Discount factor tau (float): Target update rate pi_lr (float): Learning rate for policy q_lr (float): Learning rate for critic reward_scale (float): Scaling for reward (has effect of optimal policy entropy) hidden_dim (int): Number of hidden dimensions for networks """ self.nagents = len(sa_size) self.agents = [ AttentionAgent(lr=pi_lr, hidden_dim=pol_hidden_dim, **params) for params in agent_init_params ] self.critic = AttentionCritic(sa_size, hidden_dim=critic_hidden_dim, attend_heads=attend_heads) self.target_critic = AttentionCritic(sa_size, hidden_dim=critic_hidden_dim, attend_heads=attend_heads) hard_update(self.target_critic, self.critic) self.critic_optimizer = Adam(self.critic.q_parameters(), lr=q_lr, weight_decay=1e-3) self.agent_init_params = agent_init_params self.gamma = gamma self.tau = tau self.attend_tau = attend_tau self.pi_lr = pi_lr self.q_lr = q_lr self.reward_scale = reward_scale self.pol_dev = 'cpu' # device for policies self.critic_dev = 'cpu' # device for critics self.trgt_pol_dev = 'cpu' # device for target policies self.trgt_critic_dev = 'cpu' # device for target critics self.niter = 0
def __init__(self, algo_config: List[Tuple[int, int]], gamma=0.95, tau=0.01, pi_lr=0.01, q_lr=0.01, reward_scale=10., pol_hidden_dim=128, critic_hidden_dim=128, attend_heads=4, **kwargs): """ Inputs: algo_config (List[Tuple[int, int]]): Agent types which will exist in this environment Ex. [(20, 8), (20, 2)] gamma (float): Discount factor tau (float): Target update rate pi_lr (float): Learning rate for policy q_lr (float): Learning rate for critic reward_scale (float): Scaling for reward (has effect of optimal policy entropy) hidden_dim (int): Number of hidden dimensions for networks """ print(algo_config) # Dictionary which maps agent type to its topology self.agents = [AttentionAgent(sdim, adim, lr=pi_lr, hidden_dim=pol_hidden_dim) for sdim, adim in algo_config] self.critic = AttentionCritic(algo_config, hidden_dim=critic_hidden_dim, attend_heads=attend_heads) self.target_critic = AttentionCritic(algo_config, hidden_dim=critic_hidden_dim, attend_heads=attend_heads) hard_update(self.target_critic, self.critic) self.critic_optimizer = Adam(self.critic.parameters(), lr=q_lr, weight_decay=1e-3) self.gamma = gamma self.tau = tau self.pi_lr = pi_lr self.q_lr = q_lr self.reward_scale = reward_scale self.pol_dev = 'cpu' # device for policies self.critic_dev = 'cpu' # device for critics self.trgt_pol_dev = 'cpu' # device for target policies self.trgt_critic_dev = 'cpu' # device for target critics self.niter = 0 self.init_dict = {'gamma': gamma, 'tau': tau, 'pi_lr': pi_lr, 'q_lr': q_lr, 'reward_scale': reward_scale, 'pol_hidden_dim': pol_hidden_dim, 'critic_hidden_dim': critic_hidden_dim, 'attend_heads': attend_heads, 'algo_config': algo_config}
def __init__(self, agent_init_params, sa_size, gamma=0.95, tau=0.01, pi_lr=0.01, q_lr=0.01, reward_scale=10., pol_hidden_dim=128, critic_hidden_dim=128, attend_heads=4, **kwargs): """ Inputs: agent_init_params (list of dict): List of dicts with parameters to initialize each agent num_in_pol (int): Input dimensions to policy num_out_pol (int): Output dimensions to policy sa_size (list of (int, int)): Size of state and action space for each agent """ self.nagents = len(sa_size) self.agents = [ AttentionAgent(lr=pi_lr, hidden_dim=pol_hidden_dim, **params) for params in agent_init_params ] self.critic = AttentionCritic(sa_size, hidden_dim=critic_hidden_dim, attend_heads=attend_heads) self.target_critic = AttentionCritic(sa_size, hidden_dim=critic_hidden_dim, attend_heads=attend_heads) hard_update(self.target_critic, self.critic) self.critic_optimizer = Adam(self.critic.parameters(), lr=q_lr, weight_decay=1e-3) self.agent_init_params = agent_init_params self.gamma = gamma self.tau = tau self.pi_lr = pi_lr self.q_lr = q_lr self.reward_scale = reward_scale self.niter = 0
def __init__(self, agent_init_params, sa_size, gamma=0.95, tau=0.01, pi_lr=0.01, q_lr=0.01, reward_scale=10., pol_hidden_dim=128, critic_hidden_dim=128, attend_heads=4, l1_reg=0.01, **kwargs): """ Inputs: agent_init_params (list of dict): List of dicts with parameters to initialize each agent num_in_pol (int): Input dimensions to policy num_out_pol (int): Output dimensions to policy sa_size (list of (int, int)): Size of state and action space for each agent gamma (float): Discount factor tau (float): Target update rate pi_lr (float): Learning rate for policy q_lr (float): Learning rate for critic reward_scale (float): Scaling for reward (has effect of optimal policy entropy) hidden_dim (int): Number of hidden dimensions for networks """ self.nagents = len(sa_size) self.l1_reg = l1_reg self.agents = [ AttentionAgent(lr=pi_lr, hidden_dim=pol_hidden_dim, **params) for params in agent_init_params ] widths = [] hidden_layers = [] running_width = 0 for sdim, adim in sa_size: totdim = sdim + adim running_width += totdim widths.append(running_width) hidden_layers.append(2) # def __init__(self, sa_sizes, widths, hidden_layers, selector_width, selector_depth, hidden_dim=32, **kwargs): self.critic = SelectiveAttentionCritic(sa_size, widths=widths, hidden_layers=hidden_layers, selector_width=2, selector_depth=2) self.target_critic = SelectiveAttentionCritic( sa_size, widths=widths, hidden_layers=hidden_layers, selector_width=2, selector_depth=2) hard_update(self.target_critic, self.critic) self.critic_optimizer = Adam(self.critic.parameters(), lr=q_lr, weight_decay=1e-3) self.agent_init_params = agent_init_params self.gamma = gamma self.tau = tau self.pi_lr = pi_lr self.q_lr = q_lr self.reward_scale = reward_scale self.pol_dev = 'cpu' # device for policies self.critic_dev = 'cpu' # device for critics self.trgt_pol_dev = 'cpu' # device for target policies self.trgt_critic_dev = 'cpu' # device for target critics self.niter = 0