def __init__(self, args, shared_value, share_net): seed = args.seed np.random.seed(seed) torch.manual_seed(seed) self.stop_sign = shared_value[1] self.iteration_counter = shared_value[2] self.iteration = self.iteration_counter.value self.share_net = share_net self.args = args self.env = gym.make(args.env_name) self.device = torch.device("cpu") self.actor = PolicyNet(args).to(self.device) self.Q_net1 = QNet(args).to(self.device) self.Q_net2 = QNet(args).to(self.device) self.actor_share = share_net[4] self.Q_net1_share = share_net[0] self.Q_net2_share = share_net[2] self.log_alpha_share = share_net[-1] self.alpha = np.exp(self.log_alpha_share.detach().item()) if args.alpha == 'auto' else 0 self.evaluation_interval = 50000 self.max_state_num_evaluated_in_an_episode = 500 self.episode_num_to_run = 10 self.iteration_history = [] self.evaluated_Q_mean_history=[] self.true_gamma_return_mean_history=[] # self.n_episodes_info_history = [] self.evaluated_Q_history = [] self.true_gamma_return_history = []
def __init__(self, args, shared_value, share_net): seed = args.seed np.random.seed(seed) torch.manual_seed(seed) eval_params = { 'obs_size': (160, 100), # screen size of cv2 window 'dt': 0.025, # time interval between two frames 'ego_vehicle_filter': 'vehicle.lincoln*', # filter for defining ego vehicle 'port': int(2000 + 3 * args.num_actors), # connection port 'task_mode': 'Straight', # mode of the task, [random, roundabout (only for Town03)] 'code_mode': 'test', 'max_time_episode': 100, # maximum timesteps per episode 'desired_speed': 15, # desired speed (m/s) 'max_ego_spawn_times': 100, # maximum times to spawn ego vehicle } self.stop_sign = shared_value[1] self.iteration_counter = shared_value[2] self.iteration = self.iteration_counter.value self.share_net = share_net self.args = args self.env = gym.make(args.env_name, params=eval_params) self.device = torch.device("cpu") self.actor = PolicyNet(args).to(self.device) self.Q_net1 = QNet(args).to(self.device) self.Q_net2 = QNet(args).to(self.device) self.actor_share = share_net[4] self.Q_net1_share = share_net[0] self.Q_net2_share = share_net[2] self.log_alpha_share = share_net[-1] self.alpha = np.exp(self.log_alpha_share.detach().item() ) if args.alpha == 'auto' else 0 self.evaluation_interval = 20000 self.max_state_num_evaluated_in_an_episode = 50 # 500 self.episode_num_evaluation = 5 self.episode_num_test = 5 self.time = time.time() self.list_of_n_episode_rewards_history = [] self.time_history = [] self.alpha_history = [] self.average_return_with_diff_base_history = [] self.average_reward_history = [] self.iteration_history = [] self.evaluated_Q_mean_history = [] self.evaluated_Q_std_history = [] self.true_gamma_return_mean_history = [] self.policy_entropy_history = [] self.a_std_history = [] self.a_abs_history = []
def __init__(self, args,shared_value): super(Simulation, self).__init__() seed = args.seed np.random.seed(seed) torch.manual_seed(seed) simu_params = { 'number_of_vehicles': 0, 'number_of_walkers': 0, 'obs_size': (160, 100), # screen size of cv2 window 'dt': 0.025, # time interval between two frames 'ego_vehicle_filter': 'vehicle.lincoln*', # filter for defining ego vehicle 'port': 2000, # connection port 'task_mode': 'Straight', # mode of the task, [random, roundabout (only for Town03)] 'code_mode': 'test', 'max_time_episode': 100, # maximum timesteps per episode 'desired_speed': 15, # desired speed (m/s) 'max_ego_spawn_times': 100, # maximum times to spawn ego vehicle } self.stop_sign = shared_value[1] self.args = args self.env = gym.make(args.env_name, params=simu_params) self.device = torch.device("cpu") self.load_index = self.args.max_train # self.load_index = 40000 self.actor = PolicyNet(args).to(self.device) self.actor.load_state_dict(torch.load('./'+self.args.env_name+'/method_' + str(self.args.method) + '/model/policy1_' + str(self.load_index) + '.pkl',map_location='cpu')) self.Q_net1 = QNet(args).to(self.device) self.Q_net1.load_state_dict(torch.load('./'+self.args.env_name+'/method_' + str(self.args.method) + '/model/Q1_' + str(self.load_index) + '.pkl',map_location='cpu')) if self.args.double_Q: self.Q_net2 = QNet(args).to(self.device) self.Q_net2.load_state_dict(torch.load('./'+self.args.env_name+'/method_' + str(self.args.method) + '/model/Q2_' + str(self.load_index) + '.pkl',map_location='cpu')) self.test_step = 0 self.save_interval = 10000 self.iteration = 0 self.reward_history = [] self.entropy_history = [] self.epoch_history =[] self.done_history = [] self.Q_real_history = [] self.Q_history =[] self.Q_std_history = []
def __init__(self, args, shared_value): super(Simulation, self).__init__() seed = args.seed np.random.seed(seed) torch.manual_seed(seed) self.stop_sign = shared_value[1] self.args = args self.env = gym.make(args.env_name) self.device = torch.device("cpu") self.load_index = self.args.max_train self.actor = PolicyNet(args).to(self.device) self.actor.load_state_dict( torch.load('./' + self.args.env_name + '/method_' + str(self.args.method) + '/model/policy1_' + str(self.load_index) + '.pkl', map_location='cpu')) self.Q_net1 = QNet(args).to(self.device) self.Q_net1.load_state_dict( torch.load('./' + self.args.env_name + '/method_' + str(self.args.method) + '/model/Q1_' + str(self.load_index) + '.pkl', map_location='cpu')) if self.args.double_Q: self.Q_net2 = QNet(args).to(self.device) self.Q_net2.load_state_dict( torch.load('./' + self.args.env_name + '/method_' + str(self.args.method) + '/model/Q2_' + str(self.load_index) + '.pkl', map_location='cpu')) self.test_step = 0 self.save_interval = 10000 self.iteration = 0 self.reward_history = [] self.entropy_history = [] self.epoch_history = [] self.done_history = [] self.Q_real_history = [] self.Q_history = [] self.Q_std_history = []
def __init__(self, args, shared_queue,shared_value): super(Simulation, self).__init__() seed = args.seed np.random.seed(seed) torch.manual_seed(seed) self.policy_test_queue = shared_queue[3] self.stop_sign = shared_value[1] self.args = args self.env = gym.make(args.env_name) self.device = torch.device("cpu") self.load_index = 20000 self.actor = PolicyNet(args.state_dim, args.num_hidden_cell, args.action_high, args.action_low,args.NN_type).to(self.device) self.actor.load_state_dict(torch.load('./data/method_' + str(1) + '/model/policy_' + str(self.load_index) + '.pkl')) self.Q_net1_m0 = QNet(args.state_dim, args.action_dim, args.num_hidden_cell, args.NN_type).to(self.device) self.Q_net1_m0.load_state_dict(torch.load('./data/method_' + str(0) + '/model/Q1_' + str(self.load_index) + '.pkl')) self.Q_net1_m1 = QNet(args.state_dim, args.action_dim, args.num_hidden_cell, args.NN_type).to(self.device) self.Q_net1_m1.load_state_dict(torch.load('./data/method_' + str(1) + '/model/Q1_' + str(self.load_index) + '.pkl')) self.Q_net2_m1 = QNet(args.state_dim, args.action_dim, args.num_hidden_cell, args.NN_type).to(self.device) self.Q_net2_m1.load_state_dict(torch.load('./data/method_' + str(1) + '/model/Q2_' + str(self.load_index) + '.pkl')) self.Q_net1_m2 = QNet(args.state_dim, args.action_dim, args.num_hidden_cell, args.NN_type).to(self.device) self.Q_net1_m2.load_state_dict(torch.load('./data/method_' + str(2) + '/model/Q1_' + str(self.load_index) + '.pkl')) self.test_step = 0 self.save_interval = 10000 self.iteration = 0 self.reward_history = [] self.entropy_history = [] self.epoch_history =[] self.done_history = [] self.Q_real_history = [] self.Q_m0_history =[] self.Q_m1_history = [] self.Q_m2_history = [] self.Q_std_m2_history = []
def __init__(self, args, shared_queue, shared_value): super(Test, self).__init__() seed = args.seed np.random.seed(seed) torch.manual_seed(seed) self.policy_test_queue = shared_queue[3] self.stop_sign = shared_value[1] self.args = args self.env = gym.make(args.env_name) self.device = torch.device("cpu") self.actor = PolicyNet(args.state_dim, args.num_hidden_cell, args.action_high, args.action_low, args.NN_type).to(self.device) self.test_step = 0 self.epoch_length = 1000 self.save_interval = 10000 self.iteration = 0 self.reward_history = [] self.iteration_history = []
def __init__(self, args, shared_value, share_net): super(Test, self).__init__() seed = args.seed np.random.seed(seed) torch.manual_seed(seed) test_params = { 'obs_size': (160, 100), # screen size of cv2 window 'dt': 0.025, # time interval between two frames 'ego_vehicle_filter': 'vehicle.lincoln*', # filter for defining ego vehicle 'port': int(2000 + 3 * args.num_actors), # connection port 'task_mode': 'Straight', # mode of the task, [random, roundabout (only for Town03)] 'code_mode': 'test', 'max_time_episode': 500, # maximum timesteps per episode 'desired_speed': 8, # desired speed (m/s) 'max_ego_spawn_times': 100, # maximum times to spawn ego vehicle } self.stop_sign = shared_value[1] self.iteration_counter = shared_value[2] self.iteration = self.iteration_counter.value self.args = args self.env = gym.make(args.env_name, params=test_params) self.device = torch.device("cpu") self.actor = PolicyNet(args).to(self.device) self.actor_share = share_net[0] self.log_alpha = share_net[1] self.test_step = 0 self.episode_num = 10 self.test_interval = 20000 self.start_time = time.time() self.list_of_n_episode_rewards_history = [] self.time_history = [] self.alpha_history = [] self.average_return_with_diff_base_history = [] self.average_reward_history = [] self.iteration_history = [] self.accel_history = [] self.steer_history = []
def __init__(self, args, shared_queue, shared_value, lock, i): super(Actor, self).__init__() self.agent_id = i seed = args.seed + np.int64(self.agent_id) np.random.seed(seed) torch.manual_seed(seed) self.experience_queue = shared_queue[0] self.policy_param_queue = shared_queue[1] self.q_param_queue = shared_queue[2] self.counter = shared_value[0] self.stop_sign = shared_value[1] self.lock = lock self.env = gym.make(args.env_name) self.args = args self.device = torch.device("cpu") self.actor = PolicyNet(args.state_dim, args.num_hidden_cell, args.action_high, args.action_low, args.NN_type).to(self.device) self.Q_net1 = QNet(args.state_dim, args.action_dim, args.num_hidden_cell, args.NN_type).to(self.device)
def __init__(self, args, shared_queue, shared_value, share_net, lock, i): super(Actor, self).__init__() self.agent_id = i seed = args.seed + np.int64(self.agent_id) np.random.seed(seed) torch.manual_seed(seed) self.counter = shared_value[0] self.stop_sign = shared_value[1] self.lock = lock self.env = gym.make(args.env_name) self.args = args self.experience_in_queue = [] for i in range(args.num_buffers): self.experience_in_queue.append(shared_queue[0][i]) self.device = torch.device("cpu") self.actor = PolicyNet(args).to(self.device) self.Q_net1 = QNet(args).to(self.device) self.Q_net1_share = share_net[1] self.actor_share = share_net[0]
def __init__(self, args, shared_value, share_net): seed = args.seed np.random.seed(seed) torch.manual_seed(seed) self.stop_sign = shared_value[1] self.iteration_counter = shared_value[2] self.iteration = self.iteration_counter.value self.share_net = share_net self.args = args self.env = gym.make(args.env_name) self.device = torch.device("cpu") self.actor = PolicyNet(args).to(self.device) self.Q_net1 = QNet(args).to(self.device) self.Q_net2 = QNet(args).to(self.device) self.actor_share = share_net[4] self.Q_net1_share = share_net[0] self.Q_net2_share = share_net[2] self.log_alpha_share = share_net[-1] self.alpha = np.exp(self.log_alpha_share.detach().item() ) if args.alpha == 'auto' else 0 self.evaluation_interval = 20000 self.max_state_num_evaluated_in_an_episode = 500 self.episode_num_evaluation = 5 self.episode_num_test = 5 self.time = time.time() self.list_of_n_episode_rewards_history = [] self.time_history = [] self.alpha_history = [] self.average_return_with_diff_base_history = [] self.average_reward_history = [] self.iteration_history = [] self.evaluated_Q_mean_history = [] self.evaluated_Q_std_history = [] self.true_gamma_return_mean_history = [] self.policy_entropy_history = [] self.a_std_history = [] self.a_abs_history = []
def __init__(self, args, shared_queue, shared_value, share_net, lock, i): super(Actor, self).__init__() self.agent_id = i seed = args.seed + np.int64(self.agent_id) np.random.seed(seed) torch.manual_seed(seed) actor_params = { 'obs_size': (160, 100), # screen size of cv2 window 'dt': 0.025, # time interval between two frames 'ego_vehicle_filter': 'vehicle.lincoln*', # filter for defining ego vehicle 'port': int(2000 + 3 * self.agent_id), # connection port 'task_mode': 'Straight', # mode of the task, [random, roundabout (only for Town03)] 'code_mode': 'train', 'max_time_episode': 100, # maximum timesteps per episode 'desired_speed': 15, # desired speed (m/s) 'max_ego_spawn_times': 100, # maximum times to spawn ego vehicle } self.counter = shared_value[0] self.stop_sign = shared_value[1] self.lock = lock self.env = gym.make(args.env_name, params=actor_params) self.args = args self.experience_in_queue = [] for i in range(args.num_buffers): self.experience_in_queue.append(shared_queue[0][i]) self.device = torch.device("cpu") self.actor = PolicyNet(args).to(self.device) # self.Q_net1 = QNet(args).to(self.device) #share_net = [Q_net1,Q_net1_target,Q_net2,Q_net2_target,actor,actor_target,log_alpha] #share_optimizer=[Q_net1_optimizer,Q_net2_optimizer,actor_optimizer,alpha_optimizer] self.Q_net1_share = share_net[1] self.actor_share = share_net[0]
def main(): # parameters for the gym_carla environment params = { 'display_size': 256, # screen size of bird-eye render 'obs_size': 128, # screen size of cv2 window 'dt': 0.1, # time interval between two frames 'ego_vehicle_filter': 'vehicle.lincoln*', # filter for defining ego vehicle 'port': 2000, # connection port # 'town': 'Town01', # which town to simulate 'task_mode': 'Straight', # mode of the task, [random, roundabout (only for Town03)] 'code_mode': 'test', 'max_time_episode': 5000, # maximum timesteps per episode 'desired_speed': 8, # desired speed (m/s) 'max_ego_spawn_times': 100, # maximum times to spawn ego vehicle } # Set gym-carla environment env = gym.make('carla-v0', params=params) # load net device = torch.device('cpu') args = Args() args.NN_type actor = PolicyNet(args).to(device) actor.load_state_dict(torch.load('./policy1_500000.pkl',map_location='cpu')) Q_net1 = QNet(args).to(device) Q_net1.load_state_dict(torch.load('./Q1_500000.pkl',map_location='cpu')) obs, info_dict = env.reset() info = info_dict_to_array(info_dict) state_tensor = torch.FloatTensor(obs.copy()).float().to(device) info_tensor = torch.FloatTensor(info.copy()).float().to(device) # print(env.ego.get_location()) tic = time.time() done = False ret = 0 start = carla.Location(x=env.start[0], y=env.start[1], z=0.22) end = carla.Location(x=env.dest[0], y=env.dest[1], z=0.22) if args.NN_type == "CNN": state_tensor = state_tensor.permute(2, 0, 1) while not done: tac = time.time() u, log_prob = actor.get_action(state_tensor.unsqueeze(0), info_tensor.unsqueeze(0), True) u = u.squeeze(0) obs, r, done, info = env.step(u) info = info_dict_to_array(info_dict) state_tensor = torch.FloatTensor(obs.copy()).float().to(device) if args.NN_type == "CNN": state_tensor = state_tensor.permute(2, 0, 1) info_tensor = torch.FloatTensor(info.copy()).float().to(device) ret += r cv2.imshow("camera img", obs) cv2.waitKey(1) # print(info['acceleration_t'].shape) env.world.debug.draw_point(start) env.world.debug.draw_point(end) if done: toc = time.time() print("An episode took %f s" %(toc - tic)) print("total reward is", ret) print("time steps", env.time_step) env.close() env.reset() ret = 0 # print(env.ego.get_location()) done = False
from __future__ import print_function
def main(method): params = { 'obs_size': (160, 100), # screen size of cv2 window 'dt': 0.025, # time interval between two frames 'ego_vehicle_filter': 'vehicle.lincoln*', # filter for defining ego vehicle 'port': 2000, # connection port 'task_mode': 'Straight', # mode of the task, [random, roundabout (only for Town03)] 'code_mode': 'train', 'max_time_episode': 100, # maximum timesteps per episode 'desired_speed': 15, # desired speed (m/s) 'max_ego_spawn_times': 100, # maximum times to spawn ego vehicle } args = built_parser(method=method) env = gym.make(args.env_name, params=params) state_dim = env.state_space.shape action_dim = env.action_space.shape[0] args.state_dim = state_dim args.action_dim = action_dim action_high = env.action_space.high action_low = env.action_space.low args.action_high = action_high.tolist() args.action_low = action_low.tolist() args.seed = np.random.randint(0, 30) args.init_time = time.time() num_cpu = mp.cpu_count() print(state_dim, action_dim, action_high, num_cpu) if args.alpha == 'auto' and args.target_entropy == 'auto': delta_a = np.array(args.action_high, dtype=np.float32) - np.array( args.action_low, dtype=np.float32) args.target_entropy = -1 * args.action_dim # + sum(np.log(delta_a/2)) Q_net1 = QNet(args) Q_net1.train() Q_net1.share_memory() Q_net1_target = QNet(args) Q_net1_target.train() Q_net1_target.share_memory() Q_net2 = QNet(args) Q_net2.train() Q_net2.share_memory() Q_net2_target = QNet(args) Q_net2_target.train() Q_net2_target.share_memory() actor1 = PolicyNet(args) print("Network inited") if args.code_model == "eval": actor1.load_state_dict( torch.load('./' + args.env_name + '/method_' + str(args.method) + '/model/policy_' + str(args.max_train) + '.pkl')) actor1.train() actor1.share_memory() actor1_target = PolicyNet(args) actor1_target.train() actor1_target.share_memory() actor2 = PolicyNet(args) actor2.train() actor2.share_memory() actor2_target = PolicyNet(args) actor2_target.train() actor2_target.share_memory() print("Network set") Q_net1_target.load_state_dict(Q_net1.state_dict()) Q_net2_target.load_state_dict(Q_net2.state_dict()) actor1_target.load_state_dict(actor1.state_dict()) actor2_target.load_state_dict(actor2.state_dict()) print("Network loaded!") Q_net1_optimizer = my_optim.SharedAdam(Q_net1.parameters(), lr=args.critic_lr) Q_net1_optimizer.share_memory() Q_net2_optimizer = my_optim.SharedAdam(Q_net2.parameters(), lr=args.critic_lr) Q_net2_optimizer.share_memory() actor1_optimizer = my_optim.SharedAdam(actor1.parameters(), lr=args.actor_lr) actor1_optimizer.share_memory() actor2_optimizer = my_optim.SharedAdam(actor2.parameters(), lr=args.actor_lr) actor2_optimizer.share_memory() log_alpha = torch.zeros(1, dtype=torch.float32, requires_grad=True) log_alpha.share_memory_() alpha_optimizer = my_optim.SharedAdam([log_alpha], lr=args.alpha_lr) alpha_optimizer.share_memory() print("Optimizer done") share_net = [ Q_net1, Q_net1_target, Q_net2, Q_net2_target, actor1, actor1_target, actor2, actor2_target, log_alpha ] share_optimizer = [ Q_net1_optimizer, Q_net2_optimizer, actor1_optimizer, actor2_optimizer, alpha_optimizer ] experience_in_queue = [] experience_out_queue = [] for i in range(args.num_buffers): experience_in_queue.append(Queue(maxsize=10)) experience_out_queue.append(Queue(maxsize=10)) shared_queue = [experience_in_queue, experience_out_queue] step_counter = mp.Value('i', 0) stop_sign = mp.Value('i', 0) iteration_counter = mp.Value('i', 0) shared_value = [step_counter, stop_sign, iteration_counter] lock = mp.Lock() procs = [] if args.code_model == "train": for i in range(args.num_learners): if i % 2 == 0: device = torch.device("cuda:1") else: device = torch.device("cuda:0") # device = torch.device("cpu") procs.append( Process(target=leaner_agent, args=(args, shared_queue, shared_value, share_net, share_optimizer, device, lock, i))) for i in range(args.num_actors): procs.append( Process(target=actor_agent, args=(args, shared_queue, shared_value, [actor1, Q_net1], lock, i))) for i in range(args.num_buffers): procs.append( Process(target=buffer, args=(args, shared_queue, shared_value, i))) procs.append( Process(target=evaluate_agent, args=(args, shared_value, share_net))) elif args.code_model == "simu": procs.append(Process(target=simu_agent, args=(args, shared_value))) for p in procs: p.start() for p in procs: p.join()
def __init__(self, args, shared_queue, shared_value, share_net, share_optimizer, device, lock, i): super(Learner, self).__init__() self.args = args seed = self.args.seed self.init_time = self.args.init_time np.random.seed(seed) torch.manual_seed(seed) self.agent_id = i self.experience_out_queue = [] for i in range(args.num_buffers): self.experience_out_queue.append(shared_queue[1][i]) self.stop_sign = shared_value[1] self.iteration_counter = shared_value[2] self.iteration = self.iteration_counter.value self.device = device if self.device == torch.device("cpu"): self.gpu = False else: self.gpu = True self.lock = lock self.Q_net1_share, self.Q_net1_target_share, self.Q_net2_share, self.Q_net2_target_share, self.actor1_share, \ self.actor1_target_share, self.actor2_share, self.actor2_target_share, self.log_alpha_share = share_net self.Q_net1_optimizer, self.Q_net2_optimizer, self.actor1_optimizer, self.actor2_optimizer, self.alpha_optimizer = share_optimizer self.Q_net1 = QNet(args).to(self.device) self.scheduler_Q_net1 = lr_scheduler.CosineAnnealingLR( self.Q_net1_optimizer, T_max=self.args.decay_T_max, eta_min=self.args.end_lr, last_epoch=-1) self.Q_net1.train() self.Q_net1_target = QNet(args).to(self.device) self.Q_net1_target.train() self.Q_net2 = QNet(args).to(self.device) self.scheduler_Q_net2 = lr_scheduler.CosineAnnealingLR( self.Q_net2_optimizer, T_max=self.args.decay_T_max, eta_min=self.args.end_lr, last_epoch=-1) self.Q_net2.train() self.Q_net2_target = QNet(args).to(self.device) self.Q_net2_target.train() self.actor1 = PolicyNet(args).to(self.device) self.scheduler_actor1 = lr_scheduler.CosineAnnealingLR( self.actor1_optimizer, T_max=self.args.decay_T_max, eta_min=self.args.end_lr, last_epoch=-1) self.actor1.train() self.actor1_target = PolicyNet(args).to(self.device) self.actor1_target.train() self.actor2 = PolicyNet(args).to(self.device) self.scheduler_actor2 = lr_scheduler.CosineAnnealingLR( self.actor2_optimizer, T_max=self.args.decay_T_max, eta_min=self.args.end_lr, last_epoch=-1) self.actor2.train() self.actor2_target = PolicyNet(args).to(self.device) self.actor2_target.train() self.scheduler_alpha = lr_scheduler.CosineAnnealingLR( self.alpha_optimizer, T_max=self.args.decay_T_max, eta_min=self.args.end_lr, last_epoch=-1) if self.args.alpha == 'auto': self.target_entropy = args.target_entropy else: self.alpha = torch.tensor(self.args.alpha)
def __init__(self, args): super(Application, self).__init__() seed = args.seed np.random.seed(seed) torch.manual_seed(seed) self.args = args self.device = torch.device("cpu") self.load_index = self.args.max_train self.PI_net = PINet(args).to(self.device) self.PI_net.load_state_dict( torch.load('./Net1-0_0816/PI_' + str(self.load_index) + '.pkl', map_location='cpu')) self.actor = PolicyNet(args).to(self.device) self.actor.load_state_dict( torch.load('./Net1-0_0816/policy1_' + str(self.load_index) + '.pkl', map_location='cpu')) # self.Q_net0 = QNet(args).to(self.device) # self.Q_net0.load_state_dict(torch.load('./Net1-0_0816/Q1_' + str(self.load_index) + '.pkl',map_location='cpu')) self.speed_limit_max = 20 / 3.6 self.steer_factor = 20 self.action_multiplier = np.array( [math.pi / (9 * self.steer_factor), 2], dtype=np.float32) self.safe_gap = 0. self.noise_factor = 0. self.step = 0 self.frequency = 10 self.lanekeeping_max = 5 self.lane_number = 2 self.lane_width = 3.5 self.car_length = 4.5 self.car_width = 1.855 self.road_width = self.lane_number * self.lane_width self.detect_rear_dist = 80 self.detect_front_dist = 100 self.other_length = 5.169 self.other_width = 2.392 self.position_index = 0 self.position_other = np.zeros(20, dtype=np.int) self.max_state_other = np.array([ 100, self.road_width, self.speed_limit_max, math.pi / 6, 6.5, 2.5 ], dtype=np.float32) self.max_state_ego = np.array( [self.speed_limit_max, 1, math.pi / 6, math.pi / 6, math.pi / 6, 4, self.lane_width / 2, self.road_width, self.road_width, self.speed_limit_max, self.speed_limit_max, self.lane_number - 1, 5, 2, \ math.pi / 6, math.pi / 6, math.pi / 6, math.pi / 6, math.pi / 6], dtype=np.float32) self.wheel_steer_bound = [-1.5 * math.pi, 1.5 * math.pi] self.lane_list, self.lane_center_list, self.road_angle = load_map()
def main(method): args = built_parser(method=method) env = gym.make(args.env_name) state_dim = env.observation_space.shape action_dim = env.action_space.shape[0] args.state_dim = state_dim args.action_dim = action_dim action_high = env.action_space.high action_low = env.action_space.low args.action_high = action_high.tolist() args.action_low = action_low.tolist() args.seed = np.random.randint(0, 30) args.init_time = time.time() if args.alpha == 'auto' and args.target_entropy == 'auto': delta_a = np.array(args.action_high, dtype=np.float32) - np.array( args.action_low, dtype=np.float32) args.target_entropy = -1 * args.action_dim #+ sum(np.log(delta_a/2)) Q_net1 = QNet(args) Q_net1.train() Q_net1.share_memory() Q_net1_target = QNet(args) Q_net1_target.train() Q_net1_target.share_memory() Q_net2 = QNet(args) Q_net2.train() Q_net2.share_memory() Q_net2_target = QNet(args) Q_net2_target.train() Q_net2_target.share_memory() actor1 = PolicyNet(args) actor1.train() actor1.share_memory() actor1_target = PolicyNet(args) actor1_target.train() actor1_target.share_memory() actor2 = PolicyNet(args) actor2.train() actor2.share_memory() actor2_target = PolicyNet(args) actor2_target.train() actor2_target.share_memory() Q_net1_target.load_state_dict(Q_net1.state_dict()) Q_net2_target.load_state_dict(Q_net2.state_dict()) actor1_target.load_state_dict(actor1.state_dict()) actor2_target.load_state_dict(actor2.state_dict()) Q_net1_optimizer = my_optim.SharedAdam(Q_net1.parameters(), lr=args.critic_lr) Q_net1_optimizer.share_memory() Q_net2_optimizer = my_optim.SharedAdam(Q_net2.parameters(), lr=args.critic_lr) Q_net2_optimizer.share_memory() actor1_optimizer = my_optim.SharedAdam(actor1.parameters(), lr=args.actor_lr) actor1_optimizer.share_memory() actor2_optimizer = my_optim.SharedAdam(actor2.parameters(), lr=args.actor_lr) actor2_optimizer.share_memory() log_alpha = torch.zeros(1, dtype=torch.float32, requires_grad=True) log_alpha.share_memory_() alpha_optimizer = my_optim.SharedAdam([log_alpha], lr=args.alpha_lr) alpha_optimizer.share_memory() share_net = [ Q_net1, Q_net1_target, Q_net2, Q_net2_target, actor1, actor1_target, actor2, actor2_target, log_alpha ] share_optimizer = [ Q_net1_optimizer, Q_net2_optimizer, actor1_optimizer, actor2_optimizer, alpha_optimizer ] experience_in_queue = [] experience_out_queue = [] for i in range(args.num_buffers): experience_in_queue.append(Queue(maxsize=10)) experience_out_queue.append(Queue(maxsize=10)) shared_queue = [experience_in_queue, experience_out_queue] step_counter = mp.Value('i', 0) stop_sign = mp.Value('i', 0) iteration_counter = mp.Value('i', 0) shared_value = [step_counter, stop_sign, iteration_counter] lock = mp.Lock() procs = [] if args.code_model == "train": for i in range(args.num_actors): procs.append( Process(target=actor_agent, args=(args, shared_queue, shared_value, [actor1, Q_net1], lock, i))) for i in range(args.num_buffers): procs.append( Process(target=buffer, args=(args, shared_queue, shared_value, i))) procs.append( Process(target=evaluate_agent, args=(args, shared_value, share_net))) for i in range(args.num_learners): #device = torch.device("cuda") device = torch.device("cpu") procs.append( Process(target=leaner_agent, args=(args, shared_queue, shared_value, share_net, share_optimizer, device, lock, i))) elif args.code_model == "simu": procs.append(Process(target=simu_agent, args=(args, shared_value))) for p in procs: p.start() for p in procs: p.join()