def main(method):
args = built_parser(method=method)
env = gym.make(args.env_name)
state_dim = env.observation_space.shape
action_dim = env.action_space.shape[0]
args.state_dim = state_dim
args.action_dim = action_dim
action_high = env.action_space.high
action_low = env.action_space.low
args.action_high = action_high.tolist()
args.action_low = action_low.tolist()
args.seed = np.random.randint(0, 30)
args.init_time = time.time()
if args.alpha == 'auto' and args.target_entropy == 'auto':
delta_a = np.array(args.action_high, dtype=np.float32) - np.array(
args.action_low, dtype=np.float32)
args.target_entropy = -1 * args.action_dim #+ sum(np.log(delta_a/2))
Q_net1 = QNet(args)
Q_net1.train()
Q_net1.share_memory()
Q_net1_target = QNet(args)
Q_net1_target.train()
Q_net1_target.share_memory()
Q_net2 = QNet(args)
Q_net2.train()
Q_net2.share_memory()
Q_net2_target = QNet(args)
Q_net2_target.train()
Q_net2_target.share_memory()
actor1 = PolicyNet(args)
actor1.train()
actor1.share_memory()
actor1_target = PolicyNet(args)
actor1_target.train()
actor1_target.share_memory()
actor2 = PolicyNet(args)
actor2.train()
actor2.share_memory()
actor2_target = PolicyNet(args)
actor2_target.train()
actor2_target.share_memory()
Q_net1_target.load_state_dict(Q_net1.state_dict())
Q_net2_target.load_state_dict(Q_net2.state_dict())
actor1_target.load_state_dict(actor1.state_dict())
actor2_target.load_state_dict(actor2.state_dict())
Q_net1_optimizer = my_optim.SharedAdam(Q_net1.parameters(),
lr=args.critic_lr)
Q_net1_optimizer.share_memory()
Q_net2_optimizer = my_optim.SharedAdam(Q_net2.parameters(),
lr=args.critic_lr)
Q_net2_optimizer.share_memory()
actor1_optimizer = my_optim.SharedAdam(actor1.parameters(),
lr=args.actor_lr)
actor1_optimizer.share_memory()
actor2_optimizer = my_optim.SharedAdam(actor2.parameters(),
lr=args.actor_lr)
actor2_optimizer.share_memory()
log_alpha = torch.zeros(1, dtype=torch.float32, requires_grad=True)
log_alpha.share_memory_()
alpha_optimizer = my_optim.SharedAdam([log_alpha], lr=args.alpha_lr)
alpha_optimizer.share_memory()
share_net = [
Q_net1, Q_net1_target, Q_net2, Q_net2_target, actor1, actor1_target,
actor2, actor2_target, log_alpha
]
share_optimizer = [
Q_net1_optimizer, Q_net2_optimizer, actor1_optimizer, actor2_optimizer,
alpha_optimizer
]
experience_in_queue = []
experience_out_queue = []
for i in range(args.num_buffers):
experience_in_queue.append(Queue(maxsize=10))
experience_out_queue.append(Queue(maxsize=10))
shared_queue = [experience_in_queue, experience_out_queue]
step_counter = mp.Value('i', 0)
stop_sign = mp.Value('i', 0)
iteration_counter = mp.Value('i', 0)
shared_value = [step_counter, stop_sign, iteration_counter]
lock = mp.Lock()
procs = []
if args.code_model == "train":
for i in range(args.num_actors):
procs.append(
Process(target=actor_agent,
args=(args, shared_queue, shared_value,
[actor1, Q_net1], lock, i)))
for i in range(args.num_buffers):
procs.append(
Process(target=buffer,
args=(args, shared_queue, shared_value, i)))
procs.append(
Process(target=evaluate_agent,
args=(args, shared_value, share_net)))
for i in range(args.num_learners):
#device = torch.device("cuda")
device = torch.device("cpu")
procs.append(
Process(target=leaner_agent,
args=(args, shared_queue, shared_value, share_net,
share_optimizer, device, lock, i)))
elif args.code_model == "simu":
procs.append(Process(target=simu_agent, args=(args, shared_value)))
for p in procs:
p.start()
for p in procs:
p.join()
# 2 Pole Angle -24 deg 24 deg
int(check_bound(observation[3], np.arange(-0.88, 0.88, 0.08)))
# 3 Pole Velocity At Tip -Inf Inf
]
# Create Agent
actions = range(env.action_space.n)
agent = Agent(None, (25, 25, 25), actions)
temp_agent = agent.__copy__()
# Create Network
net_size = 128
net = QNet(env.observation_space.shape[0], env.action_space.n, net_size,
device).to(device)
optimizer = optim.Adam(net.parameters(), lr=1e-3)
net.train()
ok = False
guts = 0
i_episode = 0
total = 0
loss = 0
guts_required = 100
guts_print_div = 10
big_data = [[], []]
print("Learning...")
while not ok:
# Agent learning
while guts < guts_required:
observation = env.reset()
def main(method):
params = {
'obs_size': (160, 100), # screen size of cv2 window
'dt': 0.025, # time interval between two frames
'ego_vehicle_filter':
'vehicle.lincoln*', # filter for defining ego vehicle
'port': 2000, # connection port
'task_mode':
'Straight', # mode of the task, [random, roundabout (only for Town03)]
'code_mode': 'train',
'max_time_episode': 100, # maximum timesteps per episode
'desired_speed': 15, # desired speed (m/s)
'max_ego_spawn_times': 100, # maximum times to spawn ego vehicle
}
args = built_parser(method=method)
env = gym.make(args.env_name, params=params)
state_dim = env.state_space.shape
action_dim = env.action_space.shape[0]
args.state_dim = state_dim
args.action_dim = action_dim
action_high = env.action_space.high
action_low = env.action_space.low
args.action_high = action_high.tolist()
args.action_low = action_low.tolist()
args.seed = np.random.randint(0, 30)
args.init_time = time.time()
num_cpu = mp.cpu_count()
print(state_dim, action_dim, action_high, num_cpu)
if args.alpha == 'auto' and args.target_entropy == 'auto':
delta_a = np.array(args.action_high, dtype=np.float32) - np.array(
args.action_low, dtype=np.float32)
args.target_entropy = -1 * args.action_dim # + sum(np.log(delta_a/2))
Q_net1 = QNet(args)
Q_net1.train()
Q_net1.share_memory()
Q_net1_target = QNet(args)
Q_net1_target.train()
Q_net1_target.share_memory()
Q_net2 = QNet(args)
Q_net2.train()
Q_net2.share_memory()
Q_net2_target = QNet(args)
Q_net2_target.train()
Q_net2_target.share_memory()
actor1 = PolicyNet(args)
print("Network inited")
if args.code_model == "eval":
actor1.load_state_dict(
torch.load('./' + args.env_name + '/method_' + str(args.method) +
'/model/policy_' + str(args.max_train) + '.pkl'))
actor1.train()
actor1.share_memory()
actor1_target = PolicyNet(args)
actor1_target.train()
actor1_target.share_memory()
actor2 = PolicyNet(args)
actor2.train()
actor2.share_memory()
actor2_target = PolicyNet(args)
actor2_target.train()
actor2_target.share_memory()
print("Network set")
Q_net1_target.load_state_dict(Q_net1.state_dict())
Q_net2_target.load_state_dict(Q_net2.state_dict())
actor1_target.load_state_dict(actor1.state_dict())
actor2_target.load_state_dict(actor2.state_dict())
print("Network loaded!")
Q_net1_optimizer = my_optim.SharedAdam(Q_net1.parameters(),
lr=args.critic_lr)
Q_net1_optimizer.share_memory()
Q_net2_optimizer = my_optim.SharedAdam(Q_net2.parameters(),
lr=args.critic_lr)
Q_net2_optimizer.share_memory()
actor1_optimizer = my_optim.SharedAdam(actor1.parameters(),
lr=args.actor_lr)
actor1_optimizer.share_memory()
actor2_optimizer = my_optim.SharedAdam(actor2.parameters(),
lr=args.actor_lr)
actor2_optimizer.share_memory()
log_alpha = torch.zeros(1, dtype=torch.float32, requires_grad=True)
log_alpha.share_memory_()
alpha_optimizer = my_optim.SharedAdam([log_alpha], lr=args.alpha_lr)
alpha_optimizer.share_memory()
print("Optimizer done")
share_net = [
Q_net1, Q_net1_target, Q_net2, Q_net2_target, actor1, actor1_target,
actor2, actor2_target, log_alpha
]
share_optimizer = [
Q_net1_optimizer, Q_net2_optimizer, actor1_optimizer, actor2_optimizer,
alpha_optimizer
]
experience_in_queue = []
experience_out_queue = []
for i in range(args.num_buffers):
experience_in_queue.append(Queue(maxsize=10))
experience_out_queue.append(Queue(maxsize=10))
shared_queue = [experience_in_queue, experience_out_queue]
step_counter = mp.Value('i', 0)
stop_sign = mp.Value('i', 0)
iteration_counter = mp.Value('i', 0)
shared_value = [step_counter, stop_sign, iteration_counter]
lock = mp.Lock()
procs = []
if args.code_model == "train":
for i in range(args.num_learners):
if i % 2 == 0:
device = torch.device("cuda:1")
else:
device = torch.device("cuda:0")
# device = torch.device("cpu")
procs.append(
Process(target=leaner_agent,
args=(args, shared_queue, shared_value, share_net,
share_optimizer, device, lock, i)))
for i in range(args.num_actors):
procs.append(
Process(target=actor_agent,
args=(args, shared_queue, shared_value,
[actor1, Q_net1], lock, i)))
for i in range(args.num_buffers):
procs.append(
Process(target=buffer,
args=(args, shared_queue, shared_value, i)))
procs.append(
Process(target=evaluate_agent,
args=(args, shared_value, share_net)))
elif args.code_model == "simu":
procs.append(Process(target=simu_agent, args=(args, shared_value)))
for p in procs:
p.start()
for p in procs:
p.join()
class Agent():
"""Interacts with and learns from the environment."""
def __init__(self, state_size, action_size, seed, double_agent=False,dueling_agent=False):
"""Initialize an Agent object.
Params
======
state_size (int): dimension of each state
action_size (int): dimension of each action
seed (int): random seed
double_agent(bool) : True if we want to use DDQN
dueling_agent (bool): True if we want to use Dueling
"""
self.state_size = state_size
self.action_size = action_size
self.seed = random.seed(seed)
self.double_agent=double_agent
self.dueling_agent=dueling_agent
self.qnetwork_local = QNet(state_size, action_size, seed,dueling_agent=dueling_agent).to(device)
self.qnetwork_target = QNet(state_size, action_size, seed,dueling_agent=dueling_agent).to(device)
self.optimizer = optim.RMSprop(self.qnetwork_local.parameters(), lr=LR)
# Replay memory
self.memory = ReplayBuffer(action_size, BUFFER_SIZE, BATCH_SIZE, seed)
# Initialize time step (for updating every UPDATE_EVERY steps)
self.t_step = 0
def step(self, state, action, reward, next_state, done):
# Save experience in replay memory
self.memory.add(state, action, reward, next_state, done)
# Learn every UPDATE_EVERY time steps.
self.t_step = (self.t_step + 1) % UPDATE_EVERY
if self.t_step == 0:
# If enough samples are available in memory, get random subset and learn
if len(self.memory) > BATCH_SIZE:
experiences = self.memory.sample()
self.learn(experiences, GAMMA)
def act(self, state, eps=0.):
"""Returns actions for given state as per current policy.
Params
======
state (array_like): current state
eps (float): epsilon, for epsilon-greedy action selection
"""
state = torch.from_numpy(state).float().unsqueeze(0).to(device)
self.qnetwork_local.eval()
with torch.no_grad():
action_values = self.qnetwork_local(state)
self.qnetwork_local.train()
# Epsilon-greedy action selection
if random.random() > eps:
return np.argmax(action_values.cpu().data.numpy())
else:
return random.choice(np.arange(self.action_size))
def weighted_mse_loss(self,Q_expected, Q_targets,deltas):
""" Returns the weighted mean square error between Q_expected and Q_target
Params
======
Q_expected, Q_targets : target and current guesses
deltas : weights
"""
weight =( deltas/torch.sum(deltas)*BATCH_SIZE )** (-1)
return torch.mean(weight * (Q_expected - Q_targets) ** 2)
def get_q_target(self,next_states,rewards,gamma,dones):
""" Returns the target expected Q value
Params
======
next_states : list of states we arrived in
rewards : rewards we got
gamma : discounting factor
dones : list of bool telling if the episode is done
"""
# Get max predicted Q values (for next states) from target model
if (not self.double_agent):
Q_targets_next = self.qnetwork_target(next_states).detach().max(1)[0].unsqueeze(1)
else :
indices= torch.argmax(self.qnetwork_local(next_states).detach(),1)
Q_targets_next = self.qnetwork_target(next_states).detach().gather(1,indices.unsqueeze(1))
# Compute Q targets for current states
Q_targets = rewards + (gamma * Q_targets_next * (1 - dones))
return Q_targets
def learn(self, experiences, gamma):
"""Update value parameters using given batch of experience tuples.
Params
======
experiences (Tuple[torch.Tensor]): tuple of (s, a, r, s', done) tuples
gamma (float): discount factor
"""
states, actions, rewards, next_states, dones,deltas = experiences
Q_targets = self.get_q_target(next_states,rewards,gamma,dones)
# Get expected Q values from local model
Q_expected = self.qnetwork_local(states).gather(1, actions)
# Compute loss
loss = self.weighted_mse_loss(Q_expected, Q_targets,deltas)
# Minimize the loss
self.optimizer.zero_grad()
loss.backward()
self.optimizer.step()
# ------------------- update target network ------------------- #
self.soft_update(self.qnetwork_local, self.qnetwork_target, TAU)
def soft_update(self, local_model, target_model, tau):
"""Soft update model parameters.
θ_target = τ*θ_local + (1 - τ)*θ_target
Params
======
local_model (PyTorch model): weights will be copied from
target_model (PyTorch model): weights will be copied to
tau (float): interpolation parameter
"""
for target_param, local_param in zip(target_model.parameters(), local_model.parameters()):
target_param.data.copy_(tau*local_param.data + (1.0-tau)*target_param.data)
