def main(): args = get_args() print_args(args) log_dir = create_log_dir(args) if not args.evaluate: writer = SummaryWriter(log_dir) SEED = 721 env = make_env(args) # "LaserTag-small2-v0" "SlimeVolleyPixel-v0" print(env.observation_space, env.action_space) set_global_seeds(args.seed) env.seed(args.seed) if args.evaluate: test(env, args) env.close() return train(env, args, writer) writer.export_scalars_to_json(os.path.join(log_dir, "all_scalars.json")) writer.close() env.close()
def main(): args = get_args() args.noisy = True args.double = True args.dueling = True args.prioritized_replay = True args.c51 = True args.multi_step = 3 args.load_agents = True args.num_agents = 12 args.read_model = None args.evaluate = False print_args(args) log_dir = create_log_dir(args) if not args.evaluate: writer = SummaryWriter(log_dir) env = PanicEnv(num_agents=args.num_agents, scenario_=Scenario.Two_Exits, load_agents=True, read_agents=False) set_global_seeds(args.seed) if args.evaluate: test(env, args) return train(env, args, writer) writer.export_scalars_to_json(os.path.join(log_dir, "all_scalars.json")) writer.close()
def main(): args = get_args() print_args(args) model_path = f'models/bilateral_dqn/{args.env}' os.makedirs(model_path, exist_ok=True) log_dir = create_log_dir(args) if not args.evaluate: writer = SummaryWriter(log_dir) SEED = 721 if args.num_envs == 1 or args.evaluate: env = make_env( args) # "SlimeVolley-v0", "SlimeVolleyPixel-v0" 'Pong-ram-v0' else: VectorEnv = [ DummyVectorEnv, SubprocVectorEnv ][1] # https://github.com/thu-ml/tianshou/blob/master/tianshou/env/venvs.py env = VectorEnv([lambda: make_env(args) for _ in range(args.num_envs)]) print(env.observation_space, env.action_space) set_global_seeds(args.seed) env.seed(args.seed) if args.evaluate: test(env, args, model_path) env.close() return train(env, args, writer, model_path) # writer.export_scalars_to_json(os.path.join(log_dir, "all_scalars.json")) writer.close() env.close()
def main(): args = get_args() print_args(args) if args.evaluate: if args.env == "1DStatic": env = Env1DStatic(args) elif args.env == "1DDynamic": env = Env1DDynamic_Validation(args) elif args.env == "2DStatic": env = Env2DStatic(args) elif args.env == "2DDynamic": env = Env2DDynamic_Validation(args) elif args.env == "3DStatic": env = Env3DStatic(args) elif args.env == "3DDynamic": env = Env3DDynamic_Validation(args) else: if args.env == "1DStatic": env = Env1DStatic(args) elif args.env == "1DDynamic": env = Env1DDynamic(args) elif args.env == "2DStatic": env = Env2DStatic(args) elif args.env == "2DDynamic": env = Env2DDynamic(args) elif args.env == "3DStatic": env = Env3DStatic(args) elif args.env == "3DDynamic": env = Env3DDynamic(args) datetime = time.time() save_hyperparameters(args, datetime) log_dir = create_log_dir(args) writer = SummaryWriter(log_dir) set_global_seeds(args.seed) env.seed(args.seed) if args.evaluate: validate(env, args) else: train(env, args, writer, datetime) writer.flush() writer.close() env.close()
def main(): args = get_args() print_args(args) log_dir = create_log_dir(args) if not args.evaluate: writer = SummaryWriter(log_dir) env = make_atari(args.env) env = wrap_atari_dqn(env, args) set_global_seeds(args.seed) env.seed(args.seed) if args.evaluate: test(env, args) env.close() return train(env, args, writer) writer.export_scalars_to_json(os.path.join(log_dir, "all_scalars.json")) writer.close() env.close()
def main(): args = get_args() print_args(args) log_dir = create_log_dir(args) wandb.init(project=args.wandb_project, name=args.wandb_name, notes=args.wandb_notes, config=args) env = make_atari(args.env) env = wrap_atari_dqn(env, args) set_global_seeds(args.seed) env.seed(args.seed) if args.evaluate: test(env, args) env.close() return train(env, args) env.close()
def main(): Exploiter = 'DQN' EvaluatedModel = 'NashDQN' args = get_args() # args.against_baseline=False print_args(args) env = make_env( args) # "SlimeVolley-v0", "SlimeVolleyPixel-v0" 'Pong-ram-v0' print(env.observation_space, env.action_space) model_prefix = model_metadata[args.env] exploiter = load_exploiter(env, Exploiter, args) evaluated_model = load_evaluated_model(env, EvaluatedModel, args) model_dir = "models/nash_dqn/{}/{}/".format(args.env, model_prefix) exploiter_dir = "models/nash_dqn/{}/{}/exploiter/".format( args.env, model_prefix) os.makedirs(model_dir, exist_ok=True) os.makedirs(exploiter_dir, exist_ok=True) log_dir = create_log_dir(args) if not args.evaluate: writer = SummaryWriter(log_dir) set_global_seeds(args.seed) env.seed(args.seed) # Parse all models saved during training in order filelist, epi_list = [], [] for filename in os.listdir(model_dir): if filename.endswith("dqn"): filelist.append(filename.split('_')[0] + '_') # remove '_dqn' at end epi_list.append(int(filename.split('_')[0])) sort_idx = np.argsort(epi_list).tolist() filelist = [x for _, x in sorted(zip(epi_list, filelist)) ] # sort filelist according to the sorting of epi_list epi_list.sort() # filelist.sort() will not give correct answer print(epi_list) # Evaluate/exploit all models saved during training in order eval_data = {} for f, i in zip(filelist, epi_list): print('load model: ', i, model_dir, f) # if i>17000: evaluated_model.load_model(model_dir + f, eval=True, map_location='cuda:0') exploiter_path = exploiter_dir + f r, l = exploit(env, evaluated_model, exploiter, args, exploiter_path=exploiter_path) eval_data[str(i)] = [r, l] save_dir = 'data/{}/'.format(args.env) os.makedirs(save_dir, exist_ok=True) if args.fictitious: save_dir += '/fictitious_eval_data.npy' else: save_dir += '/eval_data.npy' np.save(save_dir, eval_data) writer.close() env.close()
def train(env, args, writer): current_model = DQN(env, args).to(args.device) target_model = DQN(env, args).to(args.device) for para in target_model.parameters(): para.requires_grad = False if args.noisy: current_model.update_noisy_modules() target_model.update_noisy_modules() #target_model.eval() if args.load_model and os.path.isfile(args.load_model): load_model(current_model, args) update_target(current_model, target_model) epsilon_by_frame = epsilon_scheduler(args.eps_start, args.eps_final, args.eps_decay) beta_by_frame = beta_scheduler(args.beta_start, args.beta_frames) if args.prioritized_replay: replay_buffer = PrioritizedReplayBuffer(args.buffer_size, args.alpha) args.buffer_size = replay_buffer.it_capacity else: replay_buffer = ReplayBuffer(args.buffer_size) print_args(args) state_deque = deque(maxlen=args.multi_step) reward_deque = deque(maxlen=args.multi_step) action_deque = deque(maxlen=args.multi_step) if args.optim == 'adam': optimizer = optim.Adam(current_model.parameters(), lr=args.lr, eps=args.adam_eps, betas=(0.9, args.beta2)) elif args.optim == 'laprop': optimizer = laprop.LaProp(current_model.parameters(), lr=args.lr, betas=(0.9, args.beta2)) reward_list, length_list, loss_list = [], [], [] episode_reward = 0. episode_length = 0 prev_time = time.time() prev_frame = 1 state = env.reset() evaluation_interval = args.evaluation_interval for frame_idx in range(1, args.max_frames + 1): if args.render: env.render() if args.noisy: current_model.sample_noise() target_model.sample_noise() epsilon = epsilon_by_frame(frame_idx) action = current_model.act( torch.FloatTensor(state).to(args.device), epsilon) next_state, raw_reward, done, _ = env.step(action) if args.clip_rewards: reward = np.clip(raw_reward, -1., 1.) else: reward = raw_reward state_deque.append(state) reward_deque.append(reward) action_deque.append(action) if len(state_deque) == args.multi_step or done: n_reward = multi_step_reward(reward_deque, args.gamma) n_state = state_deque[0] n_action = action_deque[0] replay_buffer.push(n_state, n_action, n_reward, next_state, np.float32(done)) state = next_state episode_reward += raw_reward episode_length += 1 if episode_length >= 9950: while not done: _, _, done, _ = env.step(random.randrange(env.action_space.n)) if done: state = env.reset() reward_list.append(episode_reward) length_list.append(episode_length) if episode_length > 10000: print('{:.2f}'.format(episode_reward), end='') writer.add_scalar("data/episode_reward", episode_reward, frame_idx) writer.add_scalar("data/episode_length", episode_length, frame_idx) episode_reward, episode_length = 0., 0 state_deque.clear() reward_deque.clear() action_deque.clear() if len(replay_buffer ) > args.learning_start and frame_idx % args.train_freq == 0: beta = beta_by_frame(frame_idx) loss = compute_td_loss(current_model, target_model, replay_buffer, optimizer, args, beta) loss_list.append(loss.item()) writer.add_scalar("data/loss", loss.item(), frame_idx) if frame_idx % args.update_target == 0: update_target(current_model, target_model) if frame_idx % evaluation_interval == 0: if len(length_list) > 0: print_log(frame_idx, prev_frame, prev_time, reward_list, length_list, loss_list, args) reward_list.clear(), length_list.clear(), loss_list.clear() prev_frame = frame_idx prev_time = time.time() save_model(current_model, args) else: evaluation_interval += args.evaluation_interval if frame_idx % 200000 == 0: if args.adam_eps == 1.5e-4: save_model(current_model, args, name="{}_{}".format(args.optim, frame_idx)) else: save_model(current_model, args, name="{}{:.2e}_{}".format(args.optim, args.adam_eps, frame_idx)) reward_list.append(episode_reward) length_list.append(episode_length) print_log(frame_idx, prev_frame, prev_time, reward_list, length_list, loss_list, args) reward_list.clear(), length_list.clear(), loss_list.clear() prev_frame = frame_idx prev_time = time.time() save_model(current_model, args)