def __init__(self, state_size=OBS_DIM, action_size=ACT_DIM, random_seed=0):
"""Initialize an Agent object.
Params
=====
state_size (int): dimension of all observation
action_size (int): dimension of each action
random_seed (int): random seed
"""
self.state_size = state_size
self.action_size = action_size
self.seed = random.seed(random_seed)
self.epsilon = EPSILON
self.actor_local = Actor(state_size, action_size,
random_seed).to(device)
self.actor_target = Actor(state_size, action_size,
random_seed).to(device)
self.actor_optimizer = optim.Adam(self.actor_local.parameters(),
lr=LR_ACTOR)
self.critic_local = Critic(state_size, action_size,
random_seed).to(device)
self.critic_target = Critic(state_size, action_size,
random_seed).to(device)
self.critic_optimizer = optim.Adam(self.critic_local.parameters(),
lr=LR_CRITIC)
self.noise = OUNoise(action_size, random_seed)
self.memory = ReplayBuffer(BUFFER_SIZE, BATCH_SIZE, random_seed)
def __init__(self, state_size, action_size, seed):
self.state_size = state_size
self.action_size = action_size
self.seed = random.seed(seed)
self.t_step = 0 # counter for activating learning every few steps
self.running_c_loss = 0
self.running_a_loss = 0
self.training_cnt = 0
# Actor network (w/ target network)
self.actor_local = Actor(state_size, action_size, seed).to(device)
self.actor_target = Actor(state_size, action_size, seed).to(device)
self.actor_optimizer = optim.Adam(self.actor_local.parameters(),
lr=LR_ACTOR)
# Critic network (w/ target network)
self.critic_local = Critic(state_size, action_size, seed).to(device)
self.critic_target = Critic(state_size, action_size, seed).to(device)
self.critic_optimizer = optim.Adam(self.critic_local.parameters(),
lr=LR_CRITIC,
weight_decay=WEIGHT_DECAY)
# Noise process
self.noise = OUNoise(action_size, seed)
# Prioritized replay memory
self.prioritized_memory = PrioritizedMemory(BATCH_SIZE, BUFFER_SIZE,
seed)
def __init__(self,
action_size=4,
state_size=33,
num_agents=20,
max_steps=1000,
seed=0,
train_mode=True):
self.train_mode = train_mode
self.action_size = action_size
self.state_size = state_size
self.num_agents = num_agents
self.max_steps = max_steps
self.step_count = 0
self.scores = np.zeros(self.num_agents)
self.states, self.actions, self.rewards, self.next_states, self.dones = None, None, None, None, None
self.noise = OUNoise(self.action_size, seed)
self.memory = AgentMemory(batch_size=BATCH_SIZE,
buffer_size=MEMORY_BUFFER,
seed=seed)
self.actor = Actor(self.state_size, self.action_size, seed)
self.critic = Critic(self.state_size, self.action_size, seed)
self.target_actor = Actor(self.state_size, self.action_size, seed)
self.target_critic = Critic(self.state_size, self.action_size, seed)
self.actor_opt = optim.Adam(self.actor.parameters(), lr=LR_ACTOR)
self.critic_opt = optim.Adam(self.critic.parameters(),
lr=LR_CRITIC,
weight_decay=WEIGHT_DECAY)
hard_update(self.actor, self.target_actor)
hard_update(self.critic, self.target_critic)
def __init__(self, state_size, action_size, random_seed):
"""Initialize an Agent object.
Params
======
state_size (int): dimension of each state
action_size (int): dimension of each action
random_seed (int): random seed
"""
self.state_size = state_size
self.action_size = action_size
self.seed = random.seed(random_seed)
# Actor Network (w/ Target Network)
# self.actor_local = Actor(state_size, action_size, random_seed).to(device)
# self.actor_target = Actor(state_size, action_size, random_seed).to(device)
# self.actor_optimizer = optim.Adam(self.actor_local.parameters(), lr=LR_ACTOR)
# initialize Class level Actor Network
if Agent.actor_local is None:
Agent.actor_local = Actor(state_size, action_size,
random_seed).to(device)
if Agent.actor_target is None:
Agent.actor_target = Actor(state_size, action_size,
random_seed).to(device)
if Agent.actor_optimizer is None:
Agent.actor_optimizer = optim.Adam(Agent.actor_local.parameters(),
lr=LR_ACTOR)
self.actor_local = Agent.actor_local
self.actor_target = Agent.actor_target
self.actor_optimizer = Agent.actor_optimizer
# Critic Network (w/ Target Network)
# self.critic_local = Critic(state_size, action_size, random_seed).to(device)
# self.critic_target = Critic(state_size, action_size, random_seed).to(device)
# self.critic_optimizer = optim.Adam(self.critic_local.parameters(), lr=LR_CRITIC, weight_decay=WEIGHT_DECAY)
# Initilise Class levell Critic Network
if Agent.critic_local is None:
Agent.critic_local = Critic(state_size, action_size,
random_seed).to(device)
if Agent.critic_target is None:
Agent.critic_target = Critic(state_size, action_size,
random_seed).to(device)
if Agent.critic_optimizer is None:
Agent.critic_optimizer = optim.Adam(
Agent.critic_local.parameters(),
lr=LR_CRITIC,
weight_decay=WEIGHT_DECAY)
self.critic_local = Agent.critic_local
self.critic_target = Agent.critic_target
self.critic_optimizer = Agent.critic_optimizer
# Noise process
self.noise = OUNoise(action_size, random_seed)
# Replay memory - only intitialise once per class
if Agent.memory is None:
print("Initialising ReplayBuffer")
Agent.memory = ReplayBuffer(action_size, BUFFER_SIZE, BATCH_SIZE,
random_seed)
def __init__(self, nb_state, nb_action):
self.nb_state = nb_state
self.nb_action = nb_action
self.actor = Actor(self.nb_state, self.nb_action)
self.actor_target = Actor(self.nb_state, self.nb_action)
self.actor_optim = Adam(self.actor.parameters(), lr=LEARNING_RATE)
self.critic = Critic(self.nb_state, self.nb_action)
self.critic_target = Critic(self.nb_state, self.nb_action)
self.critic_optim = Adam(self.critic.parameters(), lr=LEARNING_RATE)
hard_update(self.actor_target,
self.actor) # Make sure target is with the same weight
hard_update(self.critic_target, self.critic)
#Create replay buffer
self.memory = SequentialMemory(limit=MEMORY_SIZE, window_length=1)
self.random_process = OrnsteinUhlenbeckProcess(size=nb_action,
theta=OU_THETA,
mu=OU_MU,
sigma=OU_SIGMA)
self.is_training = True
self.epsilon = 1.0
self.a_t = None
self.s_t = None
if USE_CUDA: self.cuda()
def __init__(self, state_size, action_size, seed):
self.state_size = state_size
self.action_size = action_size
self.seed = random.seed(seed)
self.t_step = 0 # counter for activating learning every few steps
self.TAU = 1e-2
self.gamma = 0.99
self.BUFFER_SIZE = int(1e6)
self.BATCH_SIZE = 1024
self.LR_CRITIC = 1e-3
self.LR_ACTOR = 1e-3
self.WEIGHT_DECAY = 0.0
self.EPSILON = 1.0
self.EPSILON_DECAY = 0.99
# Actor network (w/ target network)
self.actor_local = Actor(self.state_size, self.action_size,
seed).to(device)
self.actor_target = Actor(self.state_size, self.action_size,
seed).to(device)
self.actor_optimizer = optim.Adam(self.actor_local.parameters(),
lr=self.LR_ACTOR)
# Critic network (w/ target network)
self.critic_local = Critic(self.state_size, self.action_size,
seed).to(device)
self.critic_target = Critic(self.state_size, self.action_size,
seed).to(device)
self.critic_optimizer = optim.Adam(self.critic_local.parameters(),
lr=self.LR_CRITIC,
weight_decay=self.WEIGHT_DECAY)
# Noise process
self.noise = OUNoise(self.action_size, self.seed)
def __init__(self, state_size, action_size, random_seed):
"""
state_size (int): dimension of each state
action_size (int): dimension of each action
random_seed (int): random seed
"""
self.state_size = state_size
self.action_size = action_size
self.seed = random.seed(random_seed)
# Actor Network (w/ Target Network)
self.actor_local = Actor(state_size, action_size, random_seed).to(device)
self.actor_target = Actor(state_size, action_size, random_seed).to(device)
self.actor_optimizer = optim.Adam(self.actor_local.parameters(), lr=LR_ACTOR)
if Agent.critic_local is None:
Agent.critic_local = Critic(state_size, action_size, random_seed).to(device)
if Agent.critic_target is None:
Agent.critic_target = Critic(state_size, action_size, random_seed).to(device)
if Agent.critic_optimizer is None:
Agent.critic_optimizer = optim.Adam(Agent.critic_local.parameters(), lr=LR_CRITIC, weight_decay=WEIGHT_DECAY)
self.critic_local = Agent.critic_local
self.critic_target = Agent.critic_target
self.critic_optimizer = Agent.critic_optimizer
self.noise = OUNoise(action_size, random_seed)
# Replay memory - one per class
if Agent.memory is None:
print("Initialising ReplayBuffer")
Agent.memory = ReplayBuffer(action_size, BUFFER_SIZE, BATCH_SIZE, random_seed)
self.agent_num=len(Agent.instances)
Agent.instances.append(self)
print("Appended to Agent.instances agent {}".format(self.agent_num))
def __init__(self, state_size, action_size, random_seed):
"""
Initialize the agent object
Params:
======
state_size (int): dimension of each state
action_size (int): dimension of each action
random_seed (int): random seed
"""
self.state_size = state_size
self.action_size = action_size
self.seed = random.seed(random_seed)
self.t_step = 0 # Time steps for updating the local actor
# Actor Network (w/ Target Network)
self.actor_local = Actor(state_size, action_size, random_seed).to(device)
self.actor_target = Actor(state_size, action_size, random_seed).to(device)
self.actor_optimizer = optim.Adam(self.actor_local.parameters(), lr=LR_ACTOR)
# Critic Network (w/ Target Network)
self.critic_local = Critic(state_size, action_size, random_seed).to(device)
self.critic_target = Critic(state_size, action_size, random_seed).to(device)
self.critic_optimizer = optim.Adam(self.critic_local.parameters(), lr=LR_CRITIC, weight_decay=WEIGHT_DECAY)
# Noise process
self.noise = OUNoise(action_size, random_seed)
# Replay memory
self.memory = ReplayBuffer(action_size, BUFFER_SIZE, BATCH_SIZE, random_seed)
def __init__(self,
state_size,
action_size,
memory,
device='cpu',
params=None):
"""Initialize an Agent object.
Params
======
state_size (int): dimension of each state
action_size (int): dimension of each action
memory (obj): Memory buffer to sample
device (str): device string between cuda:0 and cpu
params (dict): hyper-parameters
"""
self.state_size = state_size
self.action_size = action_size
self.device = device
self.step_t = 0
self.update_every = params['update_every']
# Set parameters
self.gamma = params['gamma']
self.tau = params['tau']
self.seed = random.seed(params['seed'])
# Actor Network (w/ Target Network)
self.actor_local = Actor(state_size, action_size, params['seed'],
params['actor_units'][0],
params['actor_units'][1]).to(device)
self.actor_target = Actor(state_size, action_size, params['seed'],
params['actor_units'][0],
params['actor_units'][1]).to(device)
self.actor_optimizer = optim.Adam(self.actor_local.parameters(),
lr=params['lr_actor'])
# Critic Network (w/ Target Network)
self.critic_local = Critic(state_size, action_size, params['seed'],
params['critic_units'][0],
params['critic_units'][1]).to(device)
self.critic_target = Critic(state_size, action_size, params['seed'],
params['critic_units'][0],
params['critic_units'][1]).to(device)
self.critic_optimizer = optim.Adam(self.critic_local.parameters(),
lr=params['lr_critic'],
weight_decay=params['weight_decay'])
# Noise process
self.noise = OUNoise(action_size,
params['seed'],
theta=params['noise_theta'],
sigma=params['noise_sigma'])
# Replay memory
self.memory = memory
def __init__(self, state_size, action_size, random_seed, device="cpu"):
"""Initialize an Agent object.
Params
------
state_size : int
dimension of each state
action_size : int
dimension of each action
random_seed : int
random seed
"""
self.state_size = state_size
self.action_size = action_size
self.seed = random.seed(random_seed)
self.device = device
# Actor Network (w/ Target Network)
self.actor_local = Actor(state_size, action_size,
random_seed).to(self.device)
self.actor_local.apply(initialize_weights)
self.actor_target = Actor(state_size, action_size,
random_seed).to(self.device)
self.actor_target.apply(initialize_weights)
self.actor_target.eval()
self.actor_optimizer = optim.Adam(self.actor_local.parameters(),
lr=LR_ACTOR)
# Critic Network (w/ Target Network)
self.critic_local = Critic(state_size, action_size,
random_seed).to(self.device)
self.critic_local.apply(initialize_weights)
self.critic_target = Critic(state_size, action_size,
random_seed).to(self.device)
self.critic_target.apply(initialize_weights)
self.critic_target.eval()
self.critic_optimizer = optim.Adam(self.critic_local.parameters(),
lr=LR_CRITIC,
weight_decay=WEIGHT_DECAY)
# Noise process
self.noise = OUNoise(action_size,
random_seed + 1,
mu=0.,
theta=THETA,
sigma=SIGMA)
# Replay memory
self.memory = ReplayBuffer(action_size, BUFFER_SIZE, BATCH_SIZE,
random_seed + 2, self.device)
# Initialize time step (for updating every UPDATE_EVERY steps)
self.t_step = 0
def __init__(self,env_name, policy_config,device = 'cpu'):
self.device = device
self.env = gym.make(env_name) #仅仅用于设置observation
self.obs_dim = self.env.observation_space.shape[0]
if isinstance(self.env.action_space, gym.spaces.Box):
self.action_dim = self.env.action_space.shape[0]
elif isinstance(self.env.action_space, gym.spaces.Discrete):
raise TypeError('Unsupported action type')
else:
raise ValueError('unsupport action ', type(self.action_dim))
self.action_limit = self.env.action_space.high[0]
self.lr = policy_config['lr']
self.actor = Actor(self.obs_dim, self.action_dim).to(device)
self.critic = Critic(self.obs_dim, self.action_dim).to(device)
self.actor_target = deepcopy(self.actor)
self.critic_target = deepcopy(self.critic)
hard_update(self.actor_target, self.actor) # Make sure target is with the same weight
hard_update(self.critic_target, self.critic)
self.actor_optim = torch.optim.Adam(params=self.actor.parameters(), lr=0.001)
self.critic_optim = torch.optim.Adam(params=self.critic.parameters(), lr=0.001)
self.discount_factor = policy_config['discount_factor']
self.tau = 0.005
def __init__(self, state_size, action_size, random_seed):
self.actor_local = Actor(state_size, action_size,
random_seed).to(device)
self.actor_target = Actor(state_size, action_size,
random_seed).to(device)
self.actor_optimizer = optim.Adam(self.actor_local.parameters(),
lr=LR_ACTOR)
# Critic Network (w/ Target Network)
self.critic_local = Critic(state_size, action_size,
random_seed).to(device)
self.critic_target = Critic(state_size, action_size,
random_seed).to(device)
self.critic_optimizer = optim.Adam(self.critic_local.parameters(),
lr=LR_CRITIC,
weight_decay=WEIGHT_DECAY)
self.memory = ReplayBuffer(action_size, BUFFER_SIZE, BATCH_SIZE,
random_seed)
def __init__(self, state_size, action_size, random_seed, memory,
update_every, device):
"""Initialize an Agent object.
Params
======
state_size (int): dimension of each state
action_size (int): dimension of each action
random_seed (int): random seed
"""
self.state_size = state_size
self.action_size = action_size
self.seed = random.seed(random_seed)
# Actor Network (w/ Target Network)
self.actor_local = Actor(state_size, action_size,
random_seed).to(device)
self.actor_target = Actor(state_size, action_size,
random_seed).to(device)
self.actor_optimizer = optim.Adam(self.actor_local.parameters(),
lr=LR_ACTOR)
# Critic Network (w/ Target Network)
self.critic_local = Critic(state_size, action_size,
random_seed).to(device)
self.critic_target = Critic(state_size, action_size,
random_seed).to(device)
self.critic_optimizer = optim.Adam(self.critic_local.parameters(),
lr=LR_CRITIC,
weight_decay=WEIGHT_DECAY)
# Noise process
self.noise = OUNoise(action_size, random_seed)
# Replay memory
self.memory = memory
# Update weights parameters
self.update_every = update_every
self.step_count = 0
self.device = device
def __init__(self, state_size, action_size, num_agents, random_seed):
"""Initialize an Agent object.
Arguments:
state_size (int) -- dimension of each state
action_size (int) -- dimension of each action
num_agents (int) -- number of agents (brains)
random_seed (int) -- random seed
"""
self.state_size = state_size
self.action_size = action_size
self.num_agents = num_agents
self.seed = random.seed(random_seed)
self.epsilon = EPSILON
### Make neural networks (local and target) for both actor and critic, and set optimizers
# Actor Network (w/ Target Network)
self.actor_local = Actor(state_size, action_size,
random_seed).to(device)
self.actor_target = Actor(state_size, action_size,
random_seed).to(device)
self.actor_optimizer = optim.Adam(self.actor_local.parameters(),
lr=LR_ACTOR)
# Critic Network (w/ Target Network)
self.critic_local = Critic(state_size, action_size,
random_seed).to(device)
self.critic_target = Critic(state_size, action_size,
random_seed).to(device)
self.critic_optimizer = optim.Adam(self.critic_local.parameters(),
lr=LR_CRITIC,
weight_decay=WEIGHT_DECAY)
# Noise process
self.noise = OUNoise((num_agents, action_size), random_seed)
# Initialize replay memory ###
self.memory = ReplayBuffer(action_size, BUFFER_SIZE, BATCH_SIZE,
random_seed)
def __init__(self, nb_states, nb_actions, args):
# self.cuda = USE_CUDA #args.cuda
self.cuda = args.cuda
self.nb_states = nb_states
self.nb_actions = nb_actions
#Init models
#actor_kwargs = {'n_inp':self.nb_states, 'n_feature_list':[args.hidden1,args.hidden2], 'n_class':self.nb_actions}
#self.actor = MLP(**actor_kwargs)
#self.actor_target = MLP(**actor_kwargs)
#self.critic = MLP(**actor_kwargs) #TODO: actor and critic has same structure for now.
#self.critic_target = MLP(**actor_kwargs)
net_cfg = {
'hidden1': args.hidden1,
'hidden2': args.hidden2,
'init_w': args.init_w
}
self.actor = Actor(self.nb_states, self.nb_actions, **net_cfg)
self.actor_target = Actor(self.nb_states, self.nb_actions, **net_cfg)
self.critic = Critic(self.nb_states, self.nb_actions, **net_cfg)
self.critic_target = Critic(self.nb_states, self.nb_actions, **net_cfg)
self.criterion = nn.MSELoss()
if self.cuda:
self.actor = self.actor.cuda(
) # torch.nn.DataParallel(self.model).cuda() #TODO dataparallel not working
self.critic = self.critic.cuda()
self.actor_target = self.actor_target.cuda()
self.critic_target = self.critic_target.cuda()
self.criterion = self.criterion.cuda()
# Set optimizer
self.actor_optim = torch.optim.Adam(self.actor.parameters(),
lr=args.prate)
self.critic_optim = torch.optim.Adam(self.critic.parameters(),
lr=args.rate)
# Loss function
self.loss_fn = torch.nn.MSELoss(size_average=False)
hard_update(self.actor_target,
self.actor) # Make sure target is with the same weight
hard_update(self.critic_target, self.critic)
self.memory = SequentialMemory(limit=args.rmsize,
window_length=args.window_length)
self.random_process = OrnsteinUhlenbeckProcess(size=nb_actions,
theta=args.ou_theta,
mu=args.ou_mu,
sigma=args.ou_sigma)
# Hyper-parameters
self.batch_size = args.bsize
self.tau = args.tau
self.discount = args.discount
self.depsilon = 1.0 / args.epsilon
self.epsilon = 1.0
self.s_t = None # Most recent state
self.a_t = None # Most recent action
self.is_training = True
def __init__(self,
state_size,
action_size,
brain_name,
seed,
params=default_params,
device=None):
"""Initialize an Agent object.
Params
======
state_size (int): dimension of each state
action_size (int): dimension of each action
seed (int): random seed
"""
params = self._fill_params(params)
# implementation and identity
self.device = device if device is not None else torch.device(
"cuda:0" if torch.cuda.is_available() else "cpu")
self.name = params['name']
self.brain_name = brain_name
# set environment information
self.state_size = state_size
self.action_size = action_size
self.seed = random.seed(seed)
# Actor Network (w/ Target Network)
self.actor_local = Actor(state_size,
action_size,
seed,
fc1_units=params['layers_actor'][0],
fc2_units=params['layers_actor'][1]).to(
self.device)
self.actor_target = Actor(state_size,
action_size,
seed,
fc1_units=params['layers_actor'][0],
fc2_units=params['layers_actor'][1]).to(
self.device)
self.actor_optimizer = optim.Adam(self.actor_local.parameters(),
lr=params['lr_actor'])
# Critic Network (w/ Target Network)
self.critic_local = Critic(state_size,
action_size,
seed,
fcs1_units=params['layers_critic'][0],
fc2_units=params['layers_critic'][1]).to(
self.device)
self.critic_target = Critic(state_size,
action_size,
seed,
fcs1_units=params['layers_critic'][0],
fc2_units=params['layers_critic'][1]).to(
self.device)
self.critic_optimizer = optim.Adam(self.critic_local.parameters(),
lr=params['lr_critic'],
weight_decay=params['weight_decay'])
# Noise process
self.noise = OUNoise(action_size, seed)
# Replay memory
self.memory = ReplayBuffer(action_size,
params['buffer_size'],
params['batch_size'],
seed,
device=self.device)
# save params
self.params = params
def __init__(self,
state_size=None,
action_size=None,
random_seed=0,
buffer_size=int(1e6), # replay buffer size
batch_size=128, # minibatch size
gamma=0.99, # discount factor
tau=1e-3, # for soft update of target parameters
lr_actor=1e-4, # learning rate of the actor
lr_critic=1e-3, # learning rate of the critic
weight_decay=0.0001 # L2 weight decay
):
"""Initialize an Agent object.
Params
======
state_size (int): dimension of each state
action_size (int): dimension of each action
random_seed (int): random seed
"""
self.state_size = state_size
self.action_size = action_size
self.seed = random.seed(random_seed)
self.buffer_size = buffer_size # replay buffer size
self.batch_size = batch_size # minibatch size
self.gamma = gamma # discount factor
self.tau = tau # for soft update of target parameters
self.lr_actor = lr_actor # learning rate of the actor
self.lr_critic = lr_critic # learning rate of the critic
self.weight_decay = weight_decay # L2 weight decay
# initialization for first class instance
if Agent.actor_local is None:
Agent.actor_local = Actor(state_size, action_size, random_seed).to(device)
if Agent.actor_target is None:
Agent.actor_target = Actor(state_size, action_size, random_seed).to(device)
if Agent.actor_optimizer is None:
Agent.actor_optimizer = optim.Adam(Agent.actor_local.parameters(), lr=lr_actor)
# Actor Network (w/ Target Network)
self.actor_local = Agent.actor_local
self.actor_target = Agent.actor_target
self.actor_optimizer = Agent.actor_optimizer
# initialization for first class instance
if Agent.critic_local is None:
Agent.critic_local = Critic(state_size, action_size, random_seed).to(device)
if Agent.critic_target is None:
Agent.critic_target = Critic(state_size, action_size, random_seed).to(device)
if Agent.critic_optimizer is None:
Agent.critic_optimizer = optim.Adam(Agent.critic_local.parameters(), lr=lr_critic, weight_decay=weight_decay)
# Critic Network (w/ Target Network)
self.critic_local = Agent.critic_local
self.critic_target = Agent.critic_target
self.critic_optimizer = Agent.critic_optimizer
# Noise process
self.noise = OUNoise(action_size, random_seed, mu=MU, theta=THETA, sigma=SIGMA)
#TODO: Maybe self.noise = OUNoise((20, action_size), random_seed)
# Replay memory
if Agent.memory is None:
Agent.memory = ReplayBuffer(action_size, buffer_size, batch_size, random_seed)
def __init__(self, num_agents, state_size, action_size, random_seed,
buffer_size, batch_size, gamma, TAU, lr_actor, lr_critic,
weight_decay, a_hidden_sizes, c_hidden_sizes):
"""Initialize an Agent object.
Params
======
state_size (int): dimension of each state
action_size (int): dimension of each action
random_seed (int): random seed
"""
self.state_size = state_size
self.action_size = action_size
self.seed = random.seed(random_seed)
# Hyperparameters
self.BUFFER_SIZE = buffer_size
self.BATCH_SIZE = batch_size
self.GAMMA = gamma
self.TAU = TAU
self.LR_ACTOR = lr_actor
self.LR_CRITIC = lr_critic
self.WEIGHT_DECAY = weight_decay
self.ACTOR_HL_SIZE = a_hidden_sizes
self.CRITIC_HL_SIZE = c_hidden_sizes
self.num_agents = num_agents
# Actor Network (w/ Target Network)
self.actor_local_1 = Actor(state_size, action_size, random_seed,
self.ACTOR_HL_SIZE).to(device)
self.actor_target_1 = Actor(state_size, action_size, random_seed,
self.ACTOR_HL_SIZE).to(device)
self.actor_optimizer_1 = optim.Adam(self.actor_local_1.parameters(),
lr=self.LR_ACTOR)
# Critic Network (w/ Target Network)
self.critic_local_1 = Critic(state_size, action_size, random_seed,
self.CRITIC_HL_SIZE).to(device)
self.critic_target_1 = Critic(state_size, action_size, random_seed,
self.CRITIC_HL_SIZE).to(device)
self.critic_optimizer_1 = optim.Adam(self.critic_local_1.parameters(),
lr=self.LR_CRITIC,
weight_decay=self.WEIGHT_DECAY)
# Actor Network (w/ Target Network)
self.actor_local_2 = Actor(state_size, action_size, random_seed,
self.ACTOR_HL_SIZE).to(device)
self.actor_target_2 = Actor(state_size, action_size, random_seed,
self.ACTOR_HL_SIZE).to(device)
self.actor_optimizer_2 = optim.Adam(self.actor_local_2.parameters(),
lr=self.LR_ACTOR)
# Critic Network (w/ Target Network)
self.critic_local_2 = Critic(state_size, action_size, random_seed,
self.CRITIC_HL_SIZE).to(device)
self.critic_target_2 = Critic(state_size, action_size, random_seed,
self.CRITIC_HL_SIZE).to(device)
self.critic_optimizer_2 = optim.Adam(self.critic_local_2.parameters(),
lr=self.LR_CRITIC,
weight_decay=self.WEIGHT_DECAY)
# Noise process
self.noise = OUNoise(action_size, random_seed)
# Replay memory
self.memory = ReplayBuffer(action_size, self.BUFFER_SIZE,
self.BATCH_SIZE, random_seed)
def __init__(self, state_size, action_size, random_seed):
"""Initialize an Agent object.
Params
======
state_size (int): dimension of each state
action_size (int): dimension of each action
random_seed (int): random seed
"""
self.state_size = state_size
self.action_size = action_size
self.seed = random.seed(random_seed)
# Actor Network (w/ Target Network)
self.actor_local = Actor(state_size, action_size,
random_seed).to(device)
self.actor_target = Actor(state_size, action_size,
random_seed).to(device)
self.actor_optimizer = optim.Adam(self.actor_local.parameters(),
lr=LR_ACTOR)
# initialize Class level Actor Network
# if Agent.actor_local is None:
# Agent.actor_local = Actor(state_size, action_size, random_seed).to(device)
# if Agent.actor_target is None:
# Agent.actor_target = Actor(state_size, action_size, random_seed).to(device)
# if Agent.actor_optimizer is None:
# Agent.actor_optimizer = optim.Adam(Agent.actor_local.parameters(), lr=LR_ACTOR)
# self.actor_local = Agent.actor_local
# self.actor_target = Agent.actor_target
# self.actor_optimizer = Agent.actor_optimizer
# Critic Network (w/ Target Network)
# self.critic_local = Critic(state_size, action_size, random_seed).to(device)
# self.critic_target = Critic(state_size, action_size, random_seed).to(device)
# self.critic_optimizer = optim.Adam(self.critic_local.parameters(), lr=LR_CRITIC, weight_decay=WEIGHT_DECAY)
# Initilise Class levell Critic Network
if Agent.critic_local is None:
Agent.critic_local = Critic(state_size, action_size,
random_seed).to(device)
if Agent.critic_target is None:
Agent.critic_target = Critic(state_size, action_size,
random_seed).to(device)
if Agent.critic_optimizer is None:
Agent.critic_optimizer = optim.Adam(
Agent.critic_local.parameters(),
lr=LR_CRITIC,
weight_decay=WEIGHT_DECAY)
self.critic_local = Agent.critic_local
self.critic_target = Agent.critic_target
self.critic_optimizer = Agent.critic_optimizer
# Noise process
self.noise = OUNoise(action_size, random_seed)
# Replay memory - only intitialise once per class
if Agent.memory is None:
print("Initialising ReplayBuffer")
Agent.memory = ReplayBuffer(action_size, BUFFER_SIZE, BATCH_SIZE,
random_seed)
# else:
# print("Sharing ReplayBuffer %s", Agent.memory)
# Add this instances - we need to access all agent states whilst learning
self.agent_num = len(Agent.instances)
Agent.instances.append(self)
print("Appended to Agent.instances agent {}".format(self.agent_num))
def __init__(self, n_state, n_action, n_agents, random_seed, device="cpu"):
"""Initialize an Agent object.
Params
------
n_state : int
dimension of each state
n_action : int
dimension of each action
random_seed : int
random seed
device :
which device is used, cpu or cuda.
"""
self.n_state = n_state
self.n_action = n_action
self.n_agents = n_agents
self.random_seed = np.random.seed(random_seed)
self.device = device
# Networks for the first agent
# Local Actor, Local Critic, Target Actor, Target Critic
self.actor_local1 = Actor(self.n_state, self.n_action,
self.random_seed).to(self.device)
self.actor_local1.apply(initialize_weights)
self.critic_local1 = Critic(self.n_state * self.n_agents,
self.n_action * self.n_agents,
self.random_seed).to(self.device)
self.critic_local1.apply(initialize_weights)
self.actor_target1 = Actor(self.n_state, self.n_action,
self.random_seed).to(self.device)
self.actor_target1.apply(initialize_weights)
self.actor_target1.eval()
self.critic_target1 = Critic(self.n_state * self.n_agents,
self.n_action * self.n_agents,
self.random_seed).to(self.device)
self.critic_target1.apply(initialize_weights)
self.critic_target1.eval()
# Networks for the second agent
# Local Actor, Local Critic, Target Actor, Target Critic
self.actor_local2 = Actor(self.n_state, self.n_action,
self.random_seed).to(self.device)
self.actor_local2.apply(initialize_weights)
self.critic_local2 = Critic(self.n_state * self.n_agents,
self.n_action * self.n_agents,
self.random_seed).to(self.device)
self.critic_local2.apply(initialize_weights)
self.actor_target2 = Actor(self.n_state, self.n_action,
self.random_seed).to(self.device)
self.actor_target2.apply(initialize_weights)
self.actor_target2.eval()
self.critic_target2 = Critic(self.n_state * self.n_agents,
self.n_action * self.n_agents,
self.random_seed).to(self.device)
self.actor_target2.apply(initialize_weights)
self.critic_target2.eval()
# optimizers
self.actor_optimizer1 = optim.Adam(self.actor_local1.parameters(),
lr=LR_ACTOR)
self.actor_optimizer2 = optim.Adam(self.actor_local2.parameters(),
lr=LR_ACTOR)
self.critic_optimizer1 = optim.Adam(self.critic_local1.parameters(),
lr=LR_CRITIC,
weight_decay=WEIGHT_DECAY)
self.critic_optimizer2 = optim.Adam(self.critic_local2.parameters(),
lr=LR_CRITIC,
weight_decay=WEIGHT_DECAY)
# Noise process
self.noise = OUNoise(n_action * 2,
random_seed + 1,
mu=0.,
theta=THETA,
sigma=SIGMA)
# Replay Buffer
self.memory = ReplayBuffer(n_action, BUFFER_SIZE, BATCH_SIZE,
random_seed + 2, self.device)
# Initialize time step (for updating every UPDATE_EVERY steps)
self.t_step = 0