def main(): # noqa: D103 parser = argparse.ArgumentParser(description='Run DQN on Atari Breakout') parser.add_argument('--env', default='Breakout-v0', help='Atari env name') parser.add_argument('-o', '--output', default='../log/', help='Directory to save data to') parser.add_argument('--seed', default=0, type=int, help='Random seed') parser.add_argument('--gamma', default=0.99, type=float, help='Discount factor') parser.add_argument('--batch_size', default=32, type=int, help='Minibatch size') parser.add_argument('--learning_rate', default=0.0001, type=float, help='Learning rate') parser.add_argument( '--initial_epsilon', default=1.0, type=float, help='Initial exploration probability in epsilon-greedy') parser.add_argument('--final_epsilon', default=0.05, type=float, help='Final exploration probability in epsilon-greedy') parser.add_argument( '--exploration_steps', default=2000000, type=int, help= 'Number of steps over which the initial value of epsilon is linearly annealed to its final value' ) parser.add_argument( '--num_samples', default=10000000, type=int, help='Number of training samples from the environment in training') parser.add_argument('--num_frames', default=4, type=int, help='Number of frames to feed to Q-Network') parser.add_argument('--num_frames_mv', default=10, type=int, help='Number of frames to used to detect movement') parser.add_argument('--frame_width', default=84, type=int, help='Resized frame width') parser.add_argument('--frame_height', default=84, type=int, help='Resized frame height') parser.add_argument( '--replay_memory_size', default=1000000, type=int, help='Number of replay memory the agent uses for training') parser.add_argument( '--target_update_freq', default=10000, type=int, help='The frequency with which the target network is updated') parser.add_argument('--train_freq', default=4, type=int, help='The frequency of actions wrt Q-network update') parser.add_argument('--save_freq', default=200000, type=int, help='The frequency with which the network is saved') parser.add_argument('--eval_freq', default=200000, type=int, help='The frequency with which the policy is evlauted') parser.add_argument( '--num_burn_in', default=50000, type=int, help= 'Number of steps to populate the replay memory before training starts') parser.add_argument('--load_network', default=False, action='store_true', help='Load trained mode') parser.add_argument('--load_network_path', default='', help='the path to the trained mode file') parser.add_argument( '--net_mode', default='dqn', help='choose the mode of net, can be linear, dqn, duel') parser.add_argument('--max_episode_length', default=10000, type=int, help='max length of each episode') parser.add_argument('--num_episodes_at_test', default=10, type=int, help='Number of episodes the agent plays at test') parser.add_argument('--ddqn', default=False, dest='ddqn', action='store_true', help='enable ddqn') parser.add_argument('--train', default=True, dest='train', action='store_true', help='Train mode') parser.add_argument('--test', dest='train', action='store_false', help='Test mode') parser.add_argument('--no_experience', default=False, action='store_true', help='do not use experience replay') parser.add_argument('--no_target', default=False, action='store_true', help='do not use target fixing') parser.add_argument('--no_monitor', default=False, action='store_true', help='do not record video') parser.add_argument('-p', '--platform', default='rle', help='rle or atari. rle: rle; atari: gym-atari') parser.add_argument('-pl', '--perlife', default=False, action='store_true', help='use per life or not. ') parser.add_argument('-mv', '--mv_reward', default=False, action='store_true', help='use movement reward or not') parser.add_argument('-c', '--clip_reward', default=False, action='store_true', help='clip reward or not') parser.add_argument('--decay_reward', default=False, action='store_true', help='decay reward or not') parser.add_argument('--expert_memory', default=None, help='path of the expert memory') parser.add_argument( '--initial_prob_replaying_expert', default=1.0, type=float, help='Initial probability of using expert replaying memory') parser.add_argument( '--final_prob_replaying_expert', default=0.05, type=float, help='Final probability of using expert replaying memory') parser.add_argument( '--steps_replaying_expert', default=1000000, type=float, help= '# steps over which the initial prob of replaying expert memory is linearly annealed to its final value' ) parser.add_argument('--trace_dir', default='', help='the trace dir for expert') parser.add_argument('--trace2mem', default=False, action='store_true', help='convert trace to memory') parser.add_argument('--mem_dump', default='', help='the path of memory dump') args = parser.parse_args() args.output = get_output_folder(args.output, args.env) if args.trace2mem: trace2mem(args) exit(0) if args.platform == 'atari': env = gym.make(args.env) else: rom_path = 'roms/' + args.env if args.no_monitor: env = rle(rom_path, record=True, path=args.output) else: env = rle(rom_path) print("Output saved to: ", args.output) print("Args used:") print(args) # here is where you should start up a session, # create your DQN agent, create your model, etc. # then you can run your fit method. num_actions = env.action_space.n print("Game ", args.env, " #actions: ", num_actions) dqn = DQNAgent(args, num_actions) if args.train: print("Training mode.") if args.perlife: env = RLEEnvPerLifeWrapper(env) dqn.fit(env, args.num_samples, args.max_episode_length) else: print("Evaluation mode.") dqn.evaluate(env, args.num_episodes_at_test, args.max_episode_length, not args.no_monitor)
def main(): # noqa: D103 parser = argparse.ArgumentParser(description='Run DQN on Atari Space Invaders') parser.add_argument('--seed', default=10703, type=int, help='Random seed') parser.add_argument('--input_shape', default=SIZE_OF_STATE, help='Input shape') parser.add_argument('--gamma', default=0.99, help='Discount factor') # TODO experiment with this value. parser.add_argument('--epsilon', default=0.1, help='Final exploration probability in epsilon-greedy') parser.add_argument('--learning_rate', default=0.00025, help='Training learning rate.') parser.add_argument('--batch_size', default=32, type = int, help= 'Batch size of the training part') parser.add_argument('--question', type=int, default=7, help='Which hw question to run.') parser.add_argument('--evaluate', action='store_true', help='Only affects worker. Run evaluation instead of training.') parser.add_argument('--worker_epsilon', type=float, help='Only affects worker. Override epsilon to use (instead of one in file).') parser.add_argument('--skip_model_restore', action='store_true', help='Only affects worker. Use a newly initialized model instead of restoring one.') parser.add_argument('--generate_fixed_samples', action='store_true', help=('Special case execution. Generate fixed samples and close. ' + 'This is necessary to run whenever the network or action space changes.')) parser.add_argument('--ai_input_dir', default='gcloud/inputs/', help='Input directory with initialization files.') parser.add_argument('--ai_output_dir', default='gcloud/outputs/', help='Output directory for gameplay files.') parser.add_argument('--is_worker', dest='is_manager', action='store_false', help='Whether this is a worker (no training).') parser.add_argument('--is_manager', dest='is_manager', action='store_true', help='Whether this is a manager (trains).') parser.set_defaults(is_manager=True) parser.add_argument('--psc', action='store_true', help=('Only affects manager. Whether on PSC, ' + 'and should for example reduce disk usage.')) # Copied from original phillip code (run.py). for opt in CPU.full_opts(): opt.update_parser(parser) parser.add_argument("--dolphin", action="store_true", default=None, help="run dolphin") for opt in DolphinRunner.full_opts(): opt.update_parser(parser) args = parser.parse_args() # run.sh might pass these in via environment variable, so user directory # might not already be expanded. args.ai_input_dir = os.path.expanduser(args.ai_input_dir) args.ai_output_dir = os.path.expanduser(args.ai_output_dir) if args.is_manager: random.seed(args.seed) np.random.seed(args.seed) tf.set_random_seed(args.seed) do_evaluation = args.evaluate or random.random() < WORKER_EVALUATION_PROBABILITY if do_evaluation or args.generate_fixed_samples: args.cpu = EVAL_CPU_LEVEL print('OVERRIDING cpu level to: ' + str(EVAL_CPU_LEVEL)) if args.generate_fixed_samples and args.is_manager: raise Exception('Can not generate fixed samples as manager. Must use ' + '--is_worker and all other necessary flags (e.g. --iso ISO_PATH)') env = SmashEnv() if not args.is_manager: env.make(args) # Opens Dolphin. question_settings = get_question_settings(args.question, args.batch_size) online_model, online_params = create_model( input_shape=args.input_shape, num_actions=env.action_space.n, model_name='online_model', create_network_fn=question_settings['create_network_fn'], learning_rate=args.learning_rate) target_model = online_model update_target_params_ops = [] if (question_settings['target_update_freq'] is not None or question_settings['is_double_network']): target_model, target_params = create_model( input_shape=args.input_shape, num_actions=env.action_space.n, model_name='target_model', create_network_fn=question_settings['create_network_fn'], learning_rate=args.learning_rate) update_target_params_ops = [t.assign(s) for s, t in zip(online_params, target_params)] replay_memory = ReplayMemory( max_size=question_settings['replay_memory_size'], error_if_full=(not args.is_manager)) saver = tf.train.Saver(max_to_keep=None) agent = DQNAgent(online_model=online_model, target_model = target_model, memory=replay_memory, gamma=args.gamma, target_update_freq=question_settings['target_update_freq'], update_target_params_ops=update_target_params_ops, batch_size=args.batch_size, is_double_network=question_settings['is_double_network'], is_double_dqn=question_settings['is_double_dqn']) sess = tf.Session() with sess.as_default(): if args.generate_fixed_samples: print('Generating ' + str(NUM_FIXED_SAMPLES) + ' fixed samples and saving to ./' + FIXED_SAMPLES_FILENAME) print('This file is only ever used on the manager.') agent.compile(sess) fix_samples = agent.prepare_fixed_samples( env, sess, UniformRandomPolicy(env.action_space.n), NUM_FIXED_SAMPLES, MAX_EPISODE_LENGTH) env.terminate() with open(FIXED_SAMPLES_FILENAME, 'wb') as f: pickle.dump(fix_samples, f) return if args.is_manager or args.skip_model_restore: agent.compile(sess) else: saver.restore(sess, os.path.join(args.ai_input_dir, WORKER_INPUT_MODEL_FILENAME)) print('_________________') print('number_actions: ' + str(env.action_space.n)) # Worker code. if not args.is_manager: print('ai_input_dir: ' + args.ai_input_dir) print('ai_output_dir: ' + args.ai_output_dir) if do_evaluation: evaluation = agent.evaluate(env, sess, GreedyPolicy(), EVAL_EPISODES, MAX_EPISODE_LENGTH) print('Evaluation: ' + str(evaluation)) with open(FIXED_SAMPLES_FILENAME, 'rb') as fixed_samples_f: fix_samples = pickle.load(fixed_samples_f) mean_max_Q = calculate_mean_max_Q(sess, online_model, fix_samples) evaluation = evaluation + (mean_max_Q,) with open(os.path.join(args.ai_output_dir, WORKER_OUTPUT_EVALUATE_FILENAME), 'wb') as f: pickle.dump(evaluation, f) env.terminate() return worker_epsilon = args.worker_epsilon if worker_epsilon is None: with open(os.path.join(args.ai_input_dir, WORKER_INPUT_EPSILON_FILENAME)) as f: lines = f.readlines() # TODO handle unexpected lines better than just ignoring? worker_epsilon = float(lines[0]) print('Worker epsilon: ' + str(worker_epsilon)) train_policy = GreedyEpsilonPolicy(worker_epsilon) agent.play(env, sess, train_policy, total_seconds=PLAY_TOTAL_SECONDS, max_episode_length=MAX_EPISODE_LENGTH) replay_memory.save_to_file(os.path.join(args.ai_output_dir, WORKER_OUTPUT_GAMEPLAY_FILENAME)) env.terminate() return # Manager code. mprint('Loading fix samples') with open(FIXED_SAMPLES_FILENAME, 'rb') as fixed_samples_f: fix_samples = pickle.load(fixed_samples_f) evaluation_dirs = set() play_dirs = set() save_model(saver, sess, args.ai_input_dir, epsilon=1.0) epsilon_generator = LinearDecayGreedyEpsilonPolicy( 1.0, args.epsilon, TOTAL_WORKER_JOBS / 5.0) fits_so_far = 0 mprint('Begin to train (now safe to run gcloud)') mprint('Initial mean_max_q: ' + str(calculate_mean_max_Q(sess, online_model, fix_samples))) while len(play_dirs) < TOTAL_WORKER_JOBS: output_dirs = os.listdir(args.ai_output_dir) output_dirs = [os.path.join(args.ai_output_dir, x) for x in output_dirs] output_dirs = set(x for x in output_dirs if os.path.isdir(x)) new_dirs = sorted(output_dirs - evaluation_dirs - play_dirs) if len(new_dirs) == 0: time.sleep(0.1) continue new_dir = new_dirs[-1] # Most recent gameplay. evaluation_path = os.path.join(new_dir, WORKER_OUTPUT_EVALUATE_FILENAME) if os.path.isfile(evaluation_path): evaluation_dirs.add(new_dir) with open(evaluation_path, 'rb') as evaluation_file: rewards, game_lengths, mean_max_Q = pickle.load(evaluation_file) evaluation = [np.mean(rewards), np.std(rewards), np.mean(game_lengths), np.std(game_lengths), mean_max_Q] mprint('Evaluation: ' + '\t'.join(str(x) for x in evaluation)) continue memory_path = os.path.join(new_dir, WORKER_OUTPUT_GAMEPLAY_FILENAME) try: if os.path.getsize(memory_path) == 0: # TODO Figure out why this happens despite temporary directory work. # Also sometimes the file doesn't exist? Hence the try/except. mprint('Output not ready somehow: ' + memory_path) time.sleep(0.1) continue with open(memory_path, 'rb') as memory_file: worker_memories = pickle.load(memory_file) except Exception as exception: print('Error reading ' + memory_path + ': ' + str(exception.args)) time.sleep(0.1) continue for worker_memory in worker_memories: replay_memory.append(*worker_memory) if args.psc: os.remove(memory_path) play_dirs.add(new_dir) if len(play_dirs) <= NUM_BURN_IN_JOBS: mprint('Skip training because still burn in.') mprint('len(worker_memories): ' + str(len(worker_memories))) continue for _ in range(int(len(worker_memories) * FITS_PER_SINGLE_MEMORY)): agent.fit(sess, fits_so_far) fits_so_far += 1 # Partial evaluation to give frequent insight into agent progress. # Last time checked, this took ~0.1 seconds to complete. mprint('mean_max_q, len(worker_memories): ' + str(calculate_mean_max_Q(sess, online_model, fix_samples)) + ', ' + str(len(worker_memories))) # Always decrement epsilon (e.g. not just when saving model). model_epsilon = epsilon_generator.get_epsilon(decay_epsilon=True) if len(play_dirs) % SAVE_MODEL_EVERY == 0: save_model(saver, sess, args.ai_input_dir, model_epsilon)
def main(): # noqa: D103 parser = argparse.ArgumentParser(description='Run DQN on Atari Breakout') parser.add_argument('--env', default='SpaceInvaders-v0', help='Atari env name') parser.add_argument('--network_name', default='linear_q_network', type=str, help='Type of model to use') parser.add_argument('--window', default=4, type=int, help='how many frames are used each time') parser.add_argument('--new_size', default=(84, 84), type=tuple, help='new size') parser.add_argument('--batch_size', default=32, type=int, help='Batch size') parser.add_argument('--replay_buffer_size', default=750000, type=int, help='Replay buffer size') parser.add_argument('--gamma', default=0.99, type=float, help='Discount factor') parser.add_argument('--alpha', default=0.0001, type=float, help='Learning rate') parser.add_argument('--epsilon', default=0.05, type=float, help='Exploration probability for epsilon-greedy') parser.add_argument('--target_update_freq', default=10000, type=int, help='Frequency for copying weights to target network') parser.add_argument('--num_burn_in', default=50000, type=int, help='Number of prefilled samples in the replay buffer') parser.add_argument('--num_iterations', default=5000000, type=int, help='Number of overal interactions to the environment') parser.add_argument('--max_episode_length', default=200000, type=int, help='Terminate earlier for one episode') parser.add_argument('--train_freq', default=4, type=int, help='Frequency for training') parser.add_argument('--repetition_times', default=3, type=int, help='Parameter for action repetition') parser.add_argument('-o', '--output', default='atari-v0', type=str, help='Directory to save data to') parser.add_argument('--seed', default=0, type=int, help='Random seed') parser.add_argument('--experience_replay', default=False, type=bool, help='Choose whether or not to use experience replay') parser.add_argument('--train', default=True, type=bool, help='Train/Evaluate, set True if train the model') parser.add_argument('--model_path', default='/media/hongbao/Study/Courses/10703/hw2/lqn_noexp', type=str, help='specify model path to evaluation') parser.add_argument('--max_grad', default=1.0, type=float, help='Parameter for huber loss') parser.add_argument('--model_num', default=5000000, type=int, help='specify saved model number during train') parser.add_argument('--log_dir', default='log', type=str, help='specify log folder to save evaluate result') parser.add_argument('--eval_num', default=100, type=int, help='number of evaluation to run') parser.add_argument('--save_freq', default=100000, type=int, help='model save frequency') args = parser.parse_args() print("\nParameters:") for arg in vars(args): print arg, getattr(args, arg) print("") env = gym.make(args.env) num_actions = env.action_space.n # define model object preprocessor = AtariPreprocessor(args.new_size) memory = ReplayMemory(args.replay_buffer_size, args.window) # Initiating policy for both tasks (training and evaluating) policy = LinearDecayGreedyEpsilonPolicy(args.epsilon, 0, 1000000) if not args.train: '''Evaluate the model''' # check model path if args.model_path is '': print "Model path must be set when evaluate" exit(1) # specific log file to save result log_file = os.path.join(args.log_dir, args.network_name, str(args.model_num)) model_dir = os.path.join(args.model_path, args.network_name, str(args.model_num)) with tf.Session() as sess: # load model with open(model_dir + ".json", 'r') as json_file: loaded_model_json = json_file.read() q_network_online = model_from_json(loaded_model_json) q_network_target = model_from_json(loaded_model_json) sess.run(tf.global_variables_initializer()) # load weights into model q_network_online.load_weights(model_dir + ".h5") q_network_target.load_weights(model_dir + ".h5") dqn_agent = DQNAgent((q_network_online, q_network_target), preprocessor, memory, policy, num_actions, args.gamma, args.target_update_freq, args.num_burn_in, args.train_freq, args.batch_size, \ args.experience_replay, args.repetition_times, args.network_name, args.max_grad, args.env, sess) dqn_agent.evaluate(env, log_file, args.eval_num) exit(0) '''Train the model''' q_network_online = create_model(args.window, args.new_size, num_actions, args.network_name, True) q_network_target = create_model(args.window, args.new_size, num_actions, args.network_name, False) # create output dir, meant to pop up error when dir exist to avoid over written os.mkdir(os.path.join(args.output, args.network_name)) with tf.Session() as sess: dqn_agent = DQNAgent((q_network_online, q_network_target), preprocessor, memory, policy, num_actions, args.gamma, args.target_update_freq, args.num_burn_in, args.train_freq, args.batch_size, \ args.experience_replay, args.repetition_times, args.network_name, args.max_grad, args.env, sess) optimizer = tf.train.AdamOptimizer(learning_rate=args.alpha) dqn_agent.compile(optimizer, mean_huber_loss) dqn_agent.fit(env, args.num_iterations, os.path.join(args.output, args.network_name), args.save_freq, args.max_episode_length)
def main(): parser = argparse.ArgumentParser(description='Run DQN on Atari Breakout') parser.add_argument('--env', default='Breakout-v0', help='Atari env name') parser.add_argument('-o', '--output', default='atari-v0', help='Directory to save data to') parser.add_argument('--seed', default=0, type=int, help='Random seed') parser.add_argument('--mode', choices=['train', 'test'], default='test') parser.add_argument('--network', choices=['deep', 'linear'], default='deep') parser.add_argument('--method', choices=['dqn', 'double', 'dueling'], default='dqn') parser.add_argument('--monitor', type=bool, default=True) parser.add_argument('--iter', type=int, default=2400000) parser.add_argument('--test_policy', choices=['Greedy', 'GreedyEpsilon'], default='GreedyEpsilon') args = parser.parse_args() args.seed = np.random.randint(0, 1000000, 1)[0] args.weights = 'models/dqn_{}_weights_{}_{}_{}.h5f'.format( args.env, args.method, args.network, args.iter) args.monitor_path = 'tmp/dqn_{}_weights_{}_{}_{}_{}'.format( args.env, args.method, args.network, args.iter, args.test_policy) if args.mode == 'train': args.monitor = False env = gym.make(args.env) if args.monitor: env = wrappers.Monitor(env, args.monitor_path) np.random.seed(args.seed) env.seed(args.seed) args.gamma = 0.99 args.learning_rate = 0.0001 args.epsilon = 0.05 args.num_iterations = 5000000 args.batch_size = 32 args.window_length = 4 args.num_burn_in = 50000 args.target_update_freq = 10000 args.log_interval = 10000 args.model_checkpoint_interval = 10000 args.train_freq = 4 args.num_actions = env.action_space.n args.input_shape = (84, 84) args.memory_max_size = 1000000 args.output = get_output_folder(args.output, args.env) args.suffix = args.method + '_' + args.network if (args.method == 'dqn'): args.enable_double_dqn = False args.enable_dueling_network = False elif (args.method == 'double'): args.enable_double_dqn = True args.enable_dueling_network = False elif (args.method == 'dueling'): args.enable_double_dqn = False args.enable_dueling_network = True else: print('Attention! Method Worng!!!') if args.test_policy == 'Greedy': test_policy = GreedyPolicy() elif args.test_policy == 'GreedyEpsilon': test_policy = GreedyEpsilonPolicy(args.epsilon) print(args) K.tensorflow_backend.set_session(get_session()) model = create_model(args.window_length, args.input_shape, args.num_actions, args.network) # we create our preprocessor, the Ataripreprocessor will only process current frame the agent is seeing. And the sequence # preprocessor will construct the state by concatenating 3 previous frames from HistoryPreprocessor and current processed frame Processor = {} Processor['Atari'] = AtariPreprocessor(args.input_shape) Processor['History'] = HistoryPreprocessor(args.window_length) ProcessorSequence = PreprocessorSequence(Processor) # construct 84x84x4 # we create our memory for saving all experience collected during training with window length 4 memory = ReplayMemory(max_size=args.memory_max_size, input_shape=args.input_shape, window_length=args.window_length) # we use linear decay greedy epsilon policy and tune the epsilon from 1 to 0.1 during the first 100w iterations and then keep using # epsilon with 0.1 to further train the network policy = LinearDecayGreedyEpsilonPolicy(GreedyEpsilonPolicy(args.epsilon), attr_name='eps', start_value=1, end_value=0.1, num_steps=1000000) # we construct our agent and use 0.99 as our discounted factor, 32 as our batch_size. We update our model for each 4 iterations. But during first # 50000 iterations, we only collect data to the memory and don't update our model. dqn = DQNAgent(q_network=model, policy=policy, memory=memory, num_actions=args.num_actions, test_policy=test_policy, preprocessor=ProcessorSequence, gamma=args.gamma, target_update_freq=args.target_update_freq, num_burn_in=args.num_burn_in, train_freq=args.train_freq, batch_size=args.batch_size, enable_double_dqn=args.enable_double_dqn, enable_dueling_network=args.enable_dueling_network) adam = Adam(lr=args.learning_rate) dqn.compile(optimizer=adam) if args.mode == 'train': weights_filename = 'dqn_{}_weights_{}.h5f'.format( args.env, args.suffix) checkpoint_weights_filename = 'dqn_' + args.env + '_weights_' + args.suffix + '_{step}.h5f' log_filename = 'dqn_{}_log_{}.json'.format(args.env, args.suffix) log_dir = '../tensorboard_{}_log_{}'.format(args.env, args.suffix) callbacks = [ ModelIntervalCheckpoint(checkpoint_weights_filename, interval=args.model_checkpoint_interval) ] callbacks += [FileLogger(log_filename, interval=100)] callbacks += [ TensorboardStepVisualization(log_dir=log_dir, histogram_freq=1, write_graph=True, write_images=True) ] # start training # we don't apply action repetition explicitly since the game will randomly skip frame itself dqn.fit(env, callbacks=callbacks, verbose=1, num_iterations=args.num_iterations, action_repetition=1, log_interval=args.log_interval, visualize=True) dqn.save_weights(weights_filename, overwrite=True) dqn.evaluate(env, num_episodes=10, visualize=True, num_burn_in=5, action_repetition=1) elif args.mode == 'test': weights_filename = 'dqn_{}_weights_{}.h5f'.format( args.env, args.suffix) if args.weights: weights_filename = args.weights dqn.load_weights(weights_filename) dqn.evaluate(env, num_episodes=250, visualize=True, num_burn_in=5, action_repetition=1) # we upload our result to openai gym if args.monitor: env.close() gym.upload(args.monitor_path, api_key='sk_J62obX9PQg2ExrM6H9rvzQ')
def main(): # noqa: D103 parser = argparse.ArgumentParser(description='Run DQN on Atari Breakout') parser.add_argument('--env', default='Enduro-v0', help='Atari env name') parser.add_argument('--seed', default=0, type=int, help='Random seed') parser.add_argument('--model_type', default='dqn', help='Model type: linear, dqn, double_linear, double_dqn') parser.add_arguement('--mode', default='train', help='Mode: train for training, test for testing') parser.add_arguement('--memory_size', default=200000, type=int, help='Replay memory size') parser.add_arguement('--save_every', default=50000, type=int, help='Frequency for saving weights') parser.add_arguement('--max_ep_length', default=50000, type=int, help='Maximum episode length during training') parser.add_arguement('--use_target_fixing', action='store_true', help='Use target fixing') parser.add_arguement('--use_replay_memory', action='store_true', help='Use replay memory') args = parser.parse_args() # Loading the appropriate environment. env = gym.make('Enduro-v0') window = 4 input_shape = (84,84) num_actions = env.action_space.n # Limit GPU use config = tf.ConfigProto() config.gpu_options.allow_growth = True sess = tf.Session(config=config) # Set mode mode = args.mode # Set model variables # Model type to train. model_type = args.model_type # Initialize the Preprocessor, Memory, policy for training, preproc = Preprocessor() memory = ReplayMemory(args.memory_size) policy = LinearDecayGreedyEpsilonPolicy(1,0.1,1000000, num_actions) # decay epsilon from 1 to 0.1 over 1 million steps # Setting experimental parameters - details of choices specified in the write up. gamma = 0.99 target_update_freq = 10000 num_burn_in = 1000 train_freq = 0 # not using this parameter batch_size = 32 target_fix_flag = args.target_fixing replay_mem_flag = args.replay_memory save_every = args.save_every print(sess) # Create a DQN agent with the specified parameters. dqn = DQNAgent(sess, window, input_shape, num_actions, model_type, preproc, memory, policy, gamma, target_fix_flag, target_update_freq, replay_mem_flag, num_burn_in, train_freq, batch_size, save_every) # Train the model on 3-5 Million frames, with given maximum episode length. if mode == 'train': dqn.fit(env, 5000000, args.max_ep_length) elif mode == 'test': # Load the model for testing. model_file = 'saved_models_dqn/model_100000.ckpt' dqn.restore_model(model_file) # Evaluate the model. dqn.evaluate(env, 20 ,5000, 'test', lambda x: True, False, True)
def main(): # noqa: D103 parser = argparse.ArgumentParser(description='Run DQN on Atari Breakout') parser.add_argument('--env', default='Breakout-v0', help='Atari env name') parser.add_argument( '-o', '--output', default='atari-v0', help='Directory to save data to') parser.add_argument('--seed', default=0, type=int, help='Random seed') parser.add_argument('-ni', '--num_iterations', default=10, type=int, help='Num of iterations for training') parser.add_argument('-m', '--max_episode_length', default=60, type=int, help='Max episode length of a sequence') parser.add_argument('-ne', '--num_episodes', default=10, type=int, help='Num of epsidoes for evaluating') parser.add_argument('-r', '--replay_memory', default=10, type=int, help='The size of replay memory') parser.add_argument('-gamma', '--discount_factor', default=0.99, type=float, help='Discount factor of MDP') parser.add_argument('-ge', '--Greedy_epsilon', default=0.95, type=float, help='The probability to choose a greedy action') args = parser.parse_args() #args.input_shape = tuple(args.input_shape) args.output = get_output_folder(args.output, args.env) # the dirs to store results os.makedirs(args.output) os.chdir(args.output) # here is where you should start up a session, # create your DQN agent, create your model, etc. # then you can run your fit method. env = gym.make('Breakout-v0') env.reset() # Preprocess image preprocess_network = preprocessors.PreprocessorSequence('network') preprocess_memory = preprocessors.PreprocessorSequence('memory') # Policy choose Greedy = policy.GreedyEpsilonPolicy(0.95) DG = policy.LinearDecayGreedyEpsilonPolicy('attr_name', 1, 0.1, 1000000) # Create model from Atari paper model = create_model(window=4, input_shape=(84, 84), num_actions=4) # load weights location = '/' # Define tensorboard tensorboard = keras.callbacks.TensorBoard(log_dir='./logs', histogram_freq=0, write_graph=True, write_images=True) # Optimazor optimizor = Adam(lr=0.00025) # Create memory memory = core.ReplayMemory(max_size=args.replay_memory, phi_length=4, window_height=84, window_length=84, rng=np.random.RandomState(100)) agent = DQNAgent(q_network=model, target=model, preprocessor={'network': preprocess_network, 'memory': preprocess_memory}, memory=memory, policy={'Greedy': Greedy, 'DG': DG}, gamma=args.discount_factor, target_update_freq=100000, num_burn_in=args.replay_memory, train_freq=4, batch_size=32 ,callbacks=tensorboard) agent.compile(optimizer= optimizor, loss_func=objectives.mean_huber_loss) agent.init_memory(env=env, max_episode_length=30) agent.fit(env=env, num_iterations=args.num_iterations, max_episode_length=args.max_episode_length) agent.evaluate(env=env, num_episodes=args.num_episodes, max_episode_length=args.max_episode_length) # store the hyperameters file_abs = "./hypermeters" with open(file_abs, "w") as f: f.write("Num of iterations:") f.write(str(args.num_iterations) + '\n') f.write("Max epsidoe length:") f.write(str(args.max_episode_length) + '\n') f.write("Num of episodes:") f.write(str(args.num_episodes) + '\n') f.write("Replay memory:") f.write(str(args.replay_memory) + '\n') f.write("Discount factor:") f.write(str(args.discount_factor) + '\n')
def main(): # noqa: D103 parser = argparse.ArgumentParser(description='Run DQN on Atari Breakout') #parser.add_argument('--env', default='Breakout-v0', help='Atari env name') parser.add_argument('--env', default='SpaceInvaders-v0', help='Atari env name') parser.add_argument('--output', default='results', help='Directory to save data to') parser.add_argument('-l', '--isLinear', default=0, type=int, choices=range(0, 2), help='1: use linear model; 0: use deep model') parser.add_argument( '-m', '--modelType', default='q', choices=['q', 'double', 'dueling'], help= 'q: q learning; double: double q learning; dueling: dueling q learning' ) parser.add_argument( '-s', '--simple', default=0, type=int, choices=range(0, 2), help= '1: without replay or target fixing ; 0: use replay and target fixing') parser.add_argument('--seed', default=0, type=int, help='Random seed') args = parser.parse_args() #args.input_shape = tuple(args.input_shape) if not os.path.exists(args.output): os.makedirs(args.output) model_name = ('linear_' if args.isLinear else 'deep_') + args.modelType + ( '_simple' if args.simple else '') args.output = get_output_folder(args.output + '/' + model_name, args.env) env = gym.make(args.env) #env = gym.wrappers.Monitor(env, args.output) env.seed(args.seed) config = tf.ConfigProto() config.gpu_options.allow_growth = True sess = tf.Session(config=config) K.set_session(sess) K.get_session().run(tf.initialize_all_variables()) is_linear = args.isLinear agent = DQNAgent( q_network=create_model(4, (84, 84), env.action_space.n, is_linear, args.modelType), q_network2=create_model(4, (84, 84), env.action_space.n, is_linear, args.modelType), preprocessor=AtariPreprocessor((84, 84)), memory=ReplayMemory(1000000, 4), gamma=0.99, target_update_freq=10000, num_burn_in=50000, train_freq=4, batch_size=32, is_linear=is_linear, model_type=args.modelType, use_replay_and_target_fixing=(not args.simple), epsilon=0, #0.05, action_interval=4, output_path=args.output, save_freq=100000) agent.compile(lr=0.0001) agent.fit(env, 5000000) agent.load_weights() agent.evaluate(env, 100, video_path_suffix='final') env.close()