def main(): # noqa: D103
parser = argparse.ArgumentParser(description='Run DQN on Atari Breakout')
parser.add_argument('--env', default='Breakout-v0', help='Atari env name')
parser.add_argument('-o',
'--output',
default='../log/',
help='Directory to save data to')
parser.add_argument('--seed', default=0, type=int, help='Random seed')
parser.add_argument('--gamma',
default=0.99,
type=float,
help='Discount factor')
parser.add_argument('--batch_size',
default=32,
type=int,
help='Minibatch size')
parser.add_argument('--learning_rate',
default=0.0001,
type=float,
help='Learning rate')
parser.add_argument(
'--initial_epsilon',
default=1.0,
type=float,
help='Initial exploration probability in epsilon-greedy')
parser.add_argument('--final_epsilon',
default=0.05,
type=float,
help='Final exploration probability in epsilon-greedy')
parser.add_argument(
'--exploration_steps',
default=2000000,
type=int,
help=
'Number of steps over which the initial value of epsilon is linearly annealed to its final value'
)
parser.add_argument(
'--num_samples',
default=10000000,
type=int,
help='Number of training samples from the environment in training')
parser.add_argument('--num_frames',
default=4,
type=int,
help='Number of frames to feed to Q-Network')
parser.add_argument('--num_frames_mv',
default=10,
type=int,
help='Number of frames to used to detect movement')
parser.add_argument('--frame_width',
default=84,
type=int,
help='Resized frame width')
parser.add_argument('--frame_height',
default=84,
type=int,
help='Resized frame height')
parser.add_argument(
'--replay_memory_size',
default=1000000,
type=int,
help='Number of replay memory the agent uses for training')
parser.add_argument(
'--target_update_freq',
default=10000,
type=int,
help='The frequency with which the target network is updated')
parser.add_argument('--train_freq',
default=4,
type=int,
help='The frequency of actions wrt Q-network update')
parser.add_argument('--save_freq',
default=200000,
type=int,
help='The frequency with which the network is saved')
parser.add_argument('--eval_freq',
default=200000,
type=int,
help='The frequency with which the policy is evlauted')
parser.add_argument(
'--num_burn_in',
default=50000,
type=int,
help=
'Number of steps to populate the replay memory before training starts')
parser.add_argument('--load_network',
default=False,
action='store_true',
help='Load trained mode')
parser.add_argument('--load_network_path',
default='',
help='the path to the trained mode file')
parser.add_argument(
'--net_mode',
default='dqn',
help='choose the mode of net, can be linear, dqn, duel')
parser.add_argument('--max_episode_length',
default=10000,
type=int,
help='max length of each episode')
parser.add_argument('--num_episodes_at_test',
default=10,
type=int,
help='Number of episodes the agent plays at test')
parser.add_argument('--ddqn',
default=False,
dest='ddqn',
action='store_true',
help='enable ddqn')
parser.add_argument('--train',
default=True,
dest='train',
action='store_true',
help='Train mode')
parser.add_argument('--test',
dest='train',
action='store_false',
help='Test mode')
parser.add_argument('--no_experience',
default=False,
action='store_true',
help='do not use experience replay')
parser.add_argument('--no_target',
default=False,
action='store_true',
help='do not use target fixing')
parser.add_argument('--no_monitor',
default=False,
action='store_true',
help='do not record video')
parser.add_argument('-p',
'--platform',
default='rle',
help='rle or atari. rle: rle; atari: gym-atari')
parser.add_argument('-pl',
'--perlife',
default=False,
action='store_true',
help='use per life or not. ')
parser.add_argument('-mv',
'--mv_reward',
default=False,
action='store_true',
help='use movement reward or not')
parser.add_argument('-c',
'--clip_reward',
default=False,
action='store_true',
help='clip reward or not')
parser.add_argument('--decay_reward',
default=False,
action='store_true',
help='decay reward or not')
parser.add_argument('--expert_memory',
default=None,
help='path of the expert memory')
parser.add_argument(
'--initial_prob_replaying_expert',
default=1.0,
type=float,
help='Initial probability of using expert replaying memory')
parser.add_argument(
'--final_prob_replaying_expert',
default=0.05,
type=float,
help='Final probability of using expert replaying memory')
parser.add_argument(
'--steps_replaying_expert',
default=1000000,
type=float,
help=
'# steps over which the initial prob of replaying expert memory is linearly annealed to its final value'
)
parser.add_argument('--trace_dir',
default='',
help='the trace dir for expert')
parser.add_argument('--trace2mem',
default=False,
action='store_true',
help='convert trace to memory')
parser.add_argument('--mem_dump',
default='',
help='the path of memory dump')
args = parser.parse_args()
args.output = get_output_folder(args.output, args.env)
if args.trace2mem:
trace2mem(args)
exit(0)
if args.platform == 'atari':
env = gym.make(args.env)
else:
rom_path = 'roms/' + args.env
if args.no_monitor:
env = rle(rom_path, record=True, path=args.output)
else:
env = rle(rom_path)
print("Output saved to: ", args.output)
print("Args used:")
print(args)
# here is where you should start up a session,
# create your DQN agent, create your model, etc.
# then you can run your fit method.
num_actions = env.action_space.n
print("Game ", args.env, " #actions: ", num_actions)
dqn = DQNAgent(args, num_actions)
if args.train:
print("Training mode.")
if args.perlife:
env = RLEEnvPerLifeWrapper(env)
dqn.fit(env, args.num_samples, args.max_episode_length)
else:
print("Evaluation mode.")
dqn.evaluate(env, args.num_episodes_at_test, args.max_episode_length,
not args.no_monitor)
def main(): # noqa: D103
parser = argparse.ArgumentParser(description='Run DQN on Atari Space Invaders')
parser.add_argument('--seed', default=10703, type=int, help='Random seed')
parser.add_argument('--input_shape', default=SIZE_OF_STATE, help='Input shape')
parser.add_argument('--gamma', default=0.99, help='Discount factor')
# TODO experiment with this value.
parser.add_argument('--epsilon', default=0.1, help='Final exploration probability in epsilon-greedy')
parser.add_argument('--learning_rate', default=0.00025, help='Training learning rate.')
parser.add_argument('--batch_size', default=32, type = int, help=
'Batch size of the training part')
parser.add_argument('--question', type=int, default=7,
help='Which hw question to run.')
parser.add_argument('--evaluate', action='store_true',
help='Only affects worker. Run evaluation instead of training.')
parser.add_argument('--worker_epsilon', type=float,
help='Only affects worker. Override epsilon to use (instead of one in file).')
parser.add_argument('--skip_model_restore', action='store_true',
help='Only affects worker. Use a newly initialized model instead of restoring one.')
parser.add_argument('--generate_fixed_samples', action='store_true',
help=('Special case execution. Generate fixed samples and close. ' +
'This is necessary to run whenever the network or action space changes.'))
parser.add_argument('--ai_input_dir', default='gcloud/inputs/',
help='Input directory with initialization files.')
parser.add_argument('--ai_output_dir', default='gcloud/outputs/',
help='Output directory for gameplay files.')
parser.add_argument('--is_worker', dest='is_manager',
action='store_false',
help='Whether this is a worker (no training).')
parser.add_argument('--is_manager', dest='is_manager',
action='store_true',
help='Whether this is a manager (trains).')
parser.set_defaults(is_manager=True)
parser.add_argument('--psc', action='store_true',
help=('Only affects manager. Whether on PSC, ' +
'and should for example reduce disk usage.'))
# Copied from original phillip code (run.py).
for opt in CPU.full_opts():
opt.update_parser(parser)
parser.add_argument("--dolphin", action="store_true", default=None, help="run dolphin")
for opt in DolphinRunner.full_opts():
opt.update_parser(parser)
args = parser.parse_args()
# run.sh might pass these in via environment variable, so user directory
# might not already be expanded.
args.ai_input_dir = os.path.expanduser(args.ai_input_dir)
args.ai_output_dir = os.path.expanduser(args.ai_output_dir)
if args.is_manager:
random.seed(args.seed)
np.random.seed(args.seed)
tf.set_random_seed(args.seed)
do_evaluation = args.evaluate or random.random() < WORKER_EVALUATION_PROBABILITY
if do_evaluation or args.generate_fixed_samples:
args.cpu = EVAL_CPU_LEVEL
print('OVERRIDING cpu level to: ' + str(EVAL_CPU_LEVEL))
if args.generate_fixed_samples and args.is_manager:
raise Exception('Can not generate fixed samples as manager. Must use ' +
'--is_worker and all other necessary flags (e.g. --iso ISO_PATH)')
env = SmashEnv()
if not args.is_manager:
env.make(args) # Opens Dolphin.
question_settings = get_question_settings(args.question, args.batch_size)
online_model, online_params = create_model(
input_shape=args.input_shape,
num_actions=env.action_space.n, model_name='online_model',
create_network_fn=question_settings['create_network_fn'],
learning_rate=args.learning_rate)
target_model = online_model
update_target_params_ops = []
if (question_settings['target_update_freq'] is not None or
question_settings['is_double_network']):
target_model, target_params = create_model(
input_shape=args.input_shape,
num_actions=env.action_space.n, model_name='target_model',
create_network_fn=question_settings['create_network_fn'],
learning_rate=args.learning_rate)
update_target_params_ops = [t.assign(s) for s, t in zip(online_params, target_params)]
replay_memory = ReplayMemory(
max_size=question_settings['replay_memory_size'],
error_if_full=(not args.is_manager))
saver = tf.train.Saver(max_to_keep=None)
agent = DQNAgent(online_model=online_model,
target_model = target_model,
memory=replay_memory,
gamma=args.gamma,
target_update_freq=question_settings['target_update_freq'],
update_target_params_ops=update_target_params_ops,
batch_size=args.batch_size,
is_double_network=question_settings['is_double_network'],
is_double_dqn=question_settings['is_double_dqn'])
sess = tf.Session()
with sess.as_default():
if args.generate_fixed_samples:
print('Generating ' + str(NUM_FIXED_SAMPLES) + ' fixed samples and saving to ./' + FIXED_SAMPLES_FILENAME)
print('This file is only ever used on the manager.')
agent.compile(sess)
fix_samples = agent.prepare_fixed_samples(
env, sess, UniformRandomPolicy(env.action_space.n),
NUM_FIXED_SAMPLES, MAX_EPISODE_LENGTH)
env.terminate()
with open(FIXED_SAMPLES_FILENAME, 'wb') as f:
pickle.dump(fix_samples, f)
return
if args.is_manager or args.skip_model_restore:
agent.compile(sess)
else:
saver.restore(sess, os.path.join(args.ai_input_dir, WORKER_INPUT_MODEL_FILENAME))
print('_________________')
print('number_actions: ' + str(env.action_space.n))
# Worker code.
if not args.is_manager:
print('ai_input_dir: ' + args.ai_input_dir)
print('ai_output_dir: ' + args.ai_output_dir)
if do_evaluation:
evaluation = agent.evaluate(env, sess, GreedyPolicy(), EVAL_EPISODES, MAX_EPISODE_LENGTH)
print('Evaluation: ' + str(evaluation))
with open(FIXED_SAMPLES_FILENAME, 'rb') as fixed_samples_f:
fix_samples = pickle.load(fixed_samples_f)
mean_max_Q = calculate_mean_max_Q(sess, online_model, fix_samples)
evaluation = evaluation + (mean_max_Q,)
with open(os.path.join(args.ai_output_dir, WORKER_OUTPUT_EVALUATE_FILENAME), 'wb') as f:
pickle.dump(evaluation, f)
env.terminate()
return
worker_epsilon = args.worker_epsilon
if worker_epsilon is None:
with open(os.path.join(args.ai_input_dir, WORKER_INPUT_EPSILON_FILENAME)) as f:
lines = f.readlines()
# TODO handle unexpected lines better than just ignoring?
worker_epsilon = float(lines[0])
print('Worker epsilon: ' + str(worker_epsilon))
train_policy = GreedyEpsilonPolicy(worker_epsilon)
agent.play(env, sess, train_policy, total_seconds=PLAY_TOTAL_SECONDS, max_episode_length=MAX_EPISODE_LENGTH)
replay_memory.save_to_file(os.path.join(args.ai_output_dir, WORKER_OUTPUT_GAMEPLAY_FILENAME))
env.terminate()
return
# Manager code.
mprint('Loading fix samples')
with open(FIXED_SAMPLES_FILENAME, 'rb') as fixed_samples_f:
fix_samples = pickle.load(fixed_samples_f)
evaluation_dirs = set()
play_dirs = set()
save_model(saver, sess, args.ai_input_dir, epsilon=1.0)
epsilon_generator = LinearDecayGreedyEpsilonPolicy(
1.0, args.epsilon, TOTAL_WORKER_JOBS / 5.0)
fits_so_far = 0
mprint('Begin to train (now safe to run gcloud)')
mprint('Initial mean_max_q: ' + str(calculate_mean_max_Q(sess, online_model, fix_samples)))
while len(play_dirs) < TOTAL_WORKER_JOBS:
output_dirs = os.listdir(args.ai_output_dir)
output_dirs = [os.path.join(args.ai_output_dir, x) for x in output_dirs]
output_dirs = set(x for x in output_dirs if os.path.isdir(x))
new_dirs = sorted(output_dirs - evaluation_dirs - play_dirs)
if len(new_dirs) == 0:
time.sleep(0.1)
continue
new_dir = new_dirs[-1] # Most recent gameplay.
evaluation_path = os.path.join(new_dir, WORKER_OUTPUT_EVALUATE_FILENAME)
if os.path.isfile(evaluation_path):
evaluation_dirs.add(new_dir)
with open(evaluation_path, 'rb') as evaluation_file:
rewards, game_lengths, mean_max_Q = pickle.load(evaluation_file)
evaluation = [np.mean(rewards), np.std(rewards),
np.mean(game_lengths), np.std(game_lengths),
mean_max_Q]
mprint('Evaluation: ' + '\t'.join(str(x) for x in evaluation))
continue
memory_path = os.path.join(new_dir, WORKER_OUTPUT_GAMEPLAY_FILENAME)
try:
if os.path.getsize(memory_path) == 0:
# TODO Figure out why this happens despite temporary directory work.
# Also sometimes the file doesn't exist? Hence the try/except.
mprint('Output not ready somehow: ' + memory_path)
time.sleep(0.1)
continue
with open(memory_path, 'rb') as memory_file:
worker_memories = pickle.load(memory_file)
except Exception as exception:
print('Error reading ' + memory_path + ': ' + str(exception.args))
time.sleep(0.1)
continue
for worker_memory in worker_memories:
replay_memory.append(*worker_memory)
if args.psc:
os.remove(memory_path)
play_dirs.add(new_dir)
if len(play_dirs) <= NUM_BURN_IN_JOBS:
mprint('Skip training because still burn in.')
mprint('len(worker_memories): ' + str(len(worker_memories)))
continue
for _ in range(int(len(worker_memories) * FITS_PER_SINGLE_MEMORY)):
agent.fit(sess, fits_so_far)
fits_so_far += 1
# Partial evaluation to give frequent insight into agent progress.
# Last time checked, this took ~0.1 seconds to complete.
mprint('mean_max_q, len(worker_memories): ' +
str(calculate_mean_max_Q(sess, online_model, fix_samples)) +
', ' + str(len(worker_memories)))
# Always decrement epsilon (e.g. not just when saving model).
model_epsilon = epsilon_generator.get_epsilon(decay_epsilon=True)
if len(play_dirs) % SAVE_MODEL_EVERY == 0:
save_model(saver, sess, args.ai_input_dir, model_epsilon)
def main(): # noqa: D103
parser = argparse.ArgumentParser(description='Run DQN on Atari Breakout')
parser.add_argument('--env', default='SpaceInvaders-v0', help='Atari env name')
parser.add_argument('--network_name', default='linear_q_network', type=str, help='Type of model to use')
parser.add_argument('--window', default=4, type=int, help='how many frames are used each time')
parser.add_argument('--new_size', default=(84, 84), type=tuple, help='new size')
parser.add_argument('--batch_size', default=32, type=int, help='Batch size')
parser.add_argument('--replay_buffer_size', default=750000, type=int, help='Replay buffer size')
parser.add_argument('--gamma', default=0.99, type=float, help='Discount factor')
parser.add_argument('--alpha', default=0.0001, type=float, help='Learning rate')
parser.add_argument('--epsilon', default=0.05, type=float, help='Exploration probability for epsilon-greedy')
parser.add_argument('--target_update_freq', default=10000, type=int,
help='Frequency for copying weights to target network')
parser.add_argument('--num_burn_in', default=50000, type=int,
help='Number of prefilled samples in the replay buffer')
parser.add_argument('--num_iterations', default=5000000, type=int,
help='Number of overal interactions to the environment')
parser.add_argument('--max_episode_length', default=200000, type=int, help='Terminate earlier for one episode')
parser.add_argument('--train_freq', default=4, type=int, help='Frequency for training')
parser.add_argument('--repetition_times', default=3, type=int, help='Parameter for action repetition')
parser.add_argument('-o', '--output', default='atari-v0', type=str, help='Directory to save data to')
parser.add_argument('--seed', default=0, type=int, help='Random seed')
parser.add_argument('--experience_replay', default=False, type=bool,
help='Choose whether or not to use experience replay')
parser.add_argument('--train', default=True, type=bool, help='Train/Evaluate, set True if train the model')
parser.add_argument('--model_path', default='/media/hongbao/Study/Courses/10703/hw2/lqn_noexp',
type=str, help='specify model path to evaluation')
parser.add_argument('--max_grad', default=1.0, type=float, help='Parameter for huber loss')
parser.add_argument('--model_num', default=5000000, type=int, help='specify saved model number during train')
parser.add_argument('--log_dir', default='log', type=str, help='specify log folder to save evaluate result')
parser.add_argument('--eval_num', default=100, type=int, help='number of evaluation to run')
parser.add_argument('--save_freq', default=100000, type=int, help='model save frequency')
args = parser.parse_args()
print("\nParameters:")
for arg in vars(args):
print arg, getattr(args, arg)
print("")
env = gym.make(args.env)
num_actions = env.action_space.n
# define model object
preprocessor = AtariPreprocessor(args.new_size)
memory = ReplayMemory(args.replay_buffer_size, args.window)
# Initiating policy for both tasks (training and evaluating)
policy = LinearDecayGreedyEpsilonPolicy(args.epsilon, 0, 1000000)
if not args.train:
'''Evaluate the model'''
# check model path
if args.model_path is '':
print "Model path must be set when evaluate"
exit(1)
# specific log file to save result
log_file = os.path.join(args.log_dir, args.network_name, str(args.model_num))
model_dir = os.path.join(args.model_path, args.network_name, str(args.model_num))
with tf.Session() as sess:
# load model
with open(model_dir + ".json", 'r') as json_file:
loaded_model_json = json_file.read()
q_network_online = model_from_json(loaded_model_json)
q_network_target = model_from_json(loaded_model_json)
sess.run(tf.global_variables_initializer())
# load weights into model
q_network_online.load_weights(model_dir + ".h5")
q_network_target.load_weights(model_dir + ".h5")
dqn_agent = DQNAgent((q_network_online, q_network_target), preprocessor, memory, policy, num_actions,
args.gamma, args.target_update_freq, args.num_burn_in, args.train_freq,
args.batch_size, \
args.experience_replay, args.repetition_times, args.network_name, args.max_grad,
args.env, sess)
dqn_agent.evaluate(env, log_file, args.eval_num)
exit(0)
'''Train the model'''
q_network_online = create_model(args.window, args.new_size, num_actions, args.network_name, True)
q_network_target = create_model(args.window, args.new_size, num_actions, args.network_name, False)
# create output dir, meant to pop up error when dir exist to avoid over written
os.mkdir(os.path.join(args.output, args.network_name))
with tf.Session() as sess:
dqn_agent = DQNAgent((q_network_online, q_network_target), preprocessor, memory, policy, num_actions,
args.gamma, args.target_update_freq, args.num_burn_in, args.train_freq, args.batch_size, \
args.experience_replay, args.repetition_times, args.network_name, args.max_grad, args.env,
sess)
optimizer = tf.train.AdamOptimizer(learning_rate=args.alpha)
dqn_agent.compile(optimizer, mean_huber_loss)
dqn_agent.fit(env, args.num_iterations, os.path.join(args.output, args.network_name), args.save_freq,
args.max_episode_length)
def main():
parser = argparse.ArgumentParser(description='Run DQN on Atari Breakout')
parser.add_argument('--env', default='Breakout-v0', help='Atari env name')
parser.add_argument('-o',
'--output',
default='atari-v0',
help='Directory to save data to')
parser.add_argument('--seed', default=0, type=int, help='Random seed')
parser.add_argument('--mode', choices=['train', 'test'], default='test')
parser.add_argument('--network',
choices=['deep', 'linear'],
default='deep')
parser.add_argument('--method',
choices=['dqn', 'double', 'dueling'],
default='dqn')
parser.add_argument('--monitor', type=bool, default=True)
parser.add_argument('--iter', type=int, default=2400000)
parser.add_argument('--test_policy',
choices=['Greedy', 'GreedyEpsilon'],
default='GreedyEpsilon')
args = parser.parse_args()
args.seed = np.random.randint(0, 1000000, 1)[0]
args.weights = 'models/dqn_{}_weights_{}_{}_{}.h5f'.format(
args.env, args.method, args.network, args.iter)
args.monitor_path = 'tmp/dqn_{}_weights_{}_{}_{}_{}'.format(
args.env, args.method, args.network, args.iter, args.test_policy)
if args.mode == 'train':
args.monitor = False
env = gym.make(args.env)
if args.monitor:
env = wrappers.Monitor(env, args.monitor_path)
np.random.seed(args.seed)
env.seed(args.seed)
args.gamma = 0.99
args.learning_rate = 0.0001
args.epsilon = 0.05
args.num_iterations = 5000000
args.batch_size = 32
args.window_length = 4
args.num_burn_in = 50000
args.target_update_freq = 10000
args.log_interval = 10000
args.model_checkpoint_interval = 10000
args.train_freq = 4
args.num_actions = env.action_space.n
args.input_shape = (84, 84)
args.memory_max_size = 1000000
args.output = get_output_folder(args.output, args.env)
args.suffix = args.method + '_' + args.network
if (args.method == 'dqn'):
args.enable_double_dqn = False
args.enable_dueling_network = False
elif (args.method == 'double'):
args.enable_double_dqn = True
args.enable_dueling_network = False
elif (args.method == 'dueling'):
args.enable_double_dqn = False
args.enable_dueling_network = True
else:
print('Attention! Method Worng!!!')
if args.test_policy == 'Greedy':
test_policy = GreedyPolicy()
elif args.test_policy == 'GreedyEpsilon':
test_policy = GreedyEpsilonPolicy(args.epsilon)
print(args)
K.tensorflow_backend.set_session(get_session())
model = create_model(args.window_length, args.input_shape,
args.num_actions, args.network)
# we create our preprocessor, the Ataripreprocessor will only process current frame the agent is seeing. And the sequence
# preprocessor will construct the state by concatenating 3 previous frames from HistoryPreprocessor and current processed frame
Processor = {}
Processor['Atari'] = AtariPreprocessor(args.input_shape)
Processor['History'] = HistoryPreprocessor(args.window_length)
ProcessorSequence = PreprocessorSequence(Processor) # construct 84x84x4
# we create our memory for saving all experience collected during training with window length 4
memory = ReplayMemory(max_size=args.memory_max_size,
input_shape=args.input_shape,
window_length=args.window_length)
# we use linear decay greedy epsilon policy and tune the epsilon from 1 to 0.1 during the first 100w iterations and then keep using
# epsilon with 0.1 to further train the network
policy = LinearDecayGreedyEpsilonPolicy(GreedyEpsilonPolicy(args.epsilon),
attr_name='eps',
start_value=1,
end_value=0.1,
num_steps=1000000)
# we construct our agent and use 0.99 as our discounted factor, 32 as our batch_size. We update our model for each 4 iterations. But during first
# 50000 iterations, we only collect data to the memory and don't update our model.
dqn = DQNAgent(q_network=model,
policy=policy,
memory=memory,
num_actions=args.num_actions,
test_policy=test_policy,
preprocessor=ProcessorSequence,
gamma=args.gamma,
target_update_freq=args.target_update_freq,
num_burn_in=args.num_burn_in,
train_freq=args.train_freq,
batch_size=args.batch_size,
enable_double_dqn=args.enable_double_dqn,
enable_dueling_network=args.enable_dueling_network)
adam = Adam(lr=args.learning_rate)
dqn.compile(optimizer=adam)
if args.mode == 'train':
weights_filename = 'dqn_{}_weights_{}.h5f'.format(
args.env, args.suffix)
checkpoint_weights_filename = 'dqn_' + args.env + '_weights_' + args.suffix + '_{step}.h5f'
log_filename = 'dqn_{}_log_{}.json'.format(args.env, args.suffix)
log_dir = '../tensorboard_{}_log_{}'.format(args.env, args.suffix)
callbacks = [
ModelIntervalCheckpoint(checkpoint_weights_filename,
interval=args.model_checkpoint_interval)
]
callbacks += [FileLogger(log_filename, interval=100)]
callbacks += [
TensorboardStepVisualization(log_dir=log_dir,
histogram_freq=1,
write_graph=True,
write_images=True)
]
# start training
# we don't apply action repetition explicitly since the game will randomly skip frame itself
dqn.fit(env,
callbacks=callbacks,
verbose=1,
num_iterations=args.num_iterations,
action_repetition=1,
log_interval=args.log_interval,
visualize=True)
dqn.save_weights(weights_filename, overwrite=True)
dqn.evaluate(env,
num_episodes=10,
visualize=True,
num_burn_in=5,
action_repetition=1)
elif args.mode == 'test':
weights_filename = 'dqn_{}_weights_{}.h5f'.format(
args.env, args.suffix)
if args.weights:
weights_filename = args.weights
dqn.load_weights(weights_filename)
dqn.evaluate(env,
num_episodes=250,
visualize=True,
num_burn_in=5,
action_repetition=1)
# we upload our result to openai gym
if args.monitor:
env.close()
gym.upload(args.monitor_path, api_key='sk_J62obX9PQg2ExrM6H9rvzQ')
def main(): # noqa: D103
parser = argparse.ArgumentParser(description='Run DQN on Atari Breakout')
parser.add_argument('--env', default='Enduro-v0', help='Atari env name')
parser.add_argument('--seed', default=0, type=int, help='Random seed')
parser.add_argument('--model_type', default='dqn', help='Model type: linear, dqn, double_linear, double_dqn')
parser.add_arguement('--mode', default='train', help='Mode: train for training, test for testing')
parser.add_arguement('--memory_size', default=200000, type=int, help='Replay memory size')
parser.add_arguement('--save_every', default=50000, type=int, help='Frequency for saving weights')
parser.add_arguement('--max_ep_length', default=50000, type=int, help='Maximum episode length during training')
parser.add_arguement('--use_target_fixing', action='store_true', help='Use target fixing')
parser.add_arguement('--use_replay_memory', action='store_true', help='Use replay memory')
args = parser.parse_args()
# Loading the appropriate environment.
env = gym.make('Enduro-v0')
window = 4
input_shape = (84,84)
num_actions = env.action_space.n
# Limit GPU use
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
# Set mode
mode = args.mode
# Set model variables
# Model type to train.
model_type = args.model_type
# Initialize the Preprocessor, Memory, policy for training,
preproc = Preprocessor()
memory = ReplayMemory(args.memory_size)
policy = LinearDecayGreedyEpsilonPolicy(1,0.1,1000000, num_actions) # decay epsilon from 1 to 0.1 over 1 million steps
# Setting experimental parameters - details of choices specified in the write up.
gamma = 0.99
target_update_freq = 10000
num_burn_in = 1000
train_freq = 0 # not using this parameter
batch_size = 32
target_fix_flag = args.target_fixing
replay_mem_flag = args.replay_memory
save_every = args.save_every
print(sess)
# Create a DQN agent with the specified parameters.
dqn = DQNAgent(sess, window, input_shape, num_actions, model_type, preproc, memory, policy,
gamma, target_fix_flag, target_update_freq, replay_mem_flag, num_burn_in, train_freq, batch_size, save_every)
# Train the model on 3-5 Million frames, with given maximum episode length.
if mode == 'train':
dqn.fit(env, 5000000, args.max_ep_length)
elif mode == 'test':
# Load the model for testing.
model_file = 'saved_models_dqn/model_100000.ckpt'
dqn.restore_model(model_file)
# Evaluate the model.
dqn.evaluate(env, 20 ,5000, 'test', lambda x: True, False, True)
def main(): # noqa: D103
parser = argparse.ArgumentParser(description='Run DQN on Atari Breakout')
parser.add_argument('--env', default='Breakout-v0', help='Atari env name')
parser.add_argument(
'-o', '--output', default='atari-v0', help='Directory to save data to')
parser.add_argument('--seed', default=0, type=int, help='Random seed')
parser.add_argument('-ni', '--num_iterations', default=10, type=int, help='Num of iterations for training')
parser.add_argument('-m', '--max_episode_length', default=60, type=int, help='Max episode length of a sequence')
parser.add_argument('-ne', '--num_episodes', default=10, type=int, help='Num of epsidoes for evaluating')
parser.add_argument('-r', '--replay_memory', default=10, type=int, help='The size of replay memory')
parser.add_argument('-gamma', '--discount_factor', default=0.99, type=float, help='Discount factor of MDP')
parser.add_argument('-ge', '--Greedy_epsilon', default=0.95, type=float, help='The probability to choose a greedy action')
args = parser.parse_args()
#args.input_shape = tuple(args.input_shape)
args.output = get_output_folder(args.output, args.env)
# the dirs to store results
os.makedirs(args.output)
os.chdir(args.output)
# here is where you should start up a session,
# create your DQN agent, create your model, etc.
# then you can run your fit method.
env = gym.make('Breakout-v0')
env.reset()
# Preprocess image
preprocess_network = preprocessors.PreprocessorSequence('network')
preprocess_memory = preprocessors.PreprocessorSequence('memory')
# Policy choose
Greedy = policy.GreedyEpsilonPolicy(0.95)
DG = policy.LinearDecayGreedyEpsilonPolicy('attr_name', 1, 0.1, 1000000)
# Create model from Atari paper
model = create_model(window=4, input_shape=(84, 84), num_actions=4)
# load weights
location = '/'
# Define tensorboard
tensorboard = keras.callbacks.TensorBoard(log_dir='./logs', histogram_freq=0, write_graph=True, write_images=True)
# Optimazor
optimizor = Adam(lr=0.00025)
# Create memory
memory = core.ReplayMemory(max_size=args.replay_memory, phi_length=4, window_height=84, window_length=84, rng=np.random.RandomState(100))
agent = DQNAgent(q_network=model, target=model, preprocessor={'network': preprocess_network, 'memory': preprocess_memory},
memory=memory, policy={'Greedy': Greedy, 'DG': DG}, gamma=args.discount_factor, target_update_freq=100000, num_burn_in=args.replay_memory, train_freq=4, batch_size=32
,callbacks=tensorboard)
agent.compile(optimizer= optimizor, loss_func=objectives.mean_huber_loss)
agent.init_memory(env=env, max_episode_length=30)
agent.fit(env=env, num_iterations=args.num_iterations, max_episode_length=args.max_episode_length)
agent.evaluate(env=env, num_episodes=args.num_episodes, max_episode_length=args.max_episode_length)
# store the hyperameters
file_abs = "./hypermeters"
with open(file_abs, "w") as f:
f.write("Num of iterations:")
f.write(str(args.num_iterations) + '\n')
f.write("Max epsidoe length:")
f.write(str(args.max_episode_length) + '\n')
f.write("Num of episodes:")
f.write(str(args.num_episodes) + '\n')
f.write("Replay memory:")
f.write(str(args.replay_memory) + '\n')
f.write("Discount factor:")
f.write(str(args.discount_factor) + '\n')
def main(): # noqa: D103
parser = argparse.ArgumentParser(description='Run DQN on Atari Breakout')
#parser.add_argument('--env', default='Breakout-v0', help='Atari env name')
parser.add_argument('--env',
default='SpaceInvaders-v0',
help='Atari env name')
parser.add_argument('--output',
default='results',
help='Directory to save data to')
parser.add_argument('-l',
'--isLinear',
default=0,
type=int,
choices=range(0, 2),
help='1: use linear model; 0: use deep model')
parser.add_argument(
'-m',
'--modelType',
default='q',
choices=['q', 'double', 'dueling'],
help=
'q: q learning; double: double q learning; dueling: dueling q learning'
)
parser.add_argument(
'-s',
'--simple',
default=0,
type=int,
choices=range(0, 2),
help=
'1: without replay or target fixing ; 0: use replay and target fixing')
parser.add_argument('--seed', default=0, type=int, help='Random seed')
args = parser.parse_args()
#args.input_shape = tuple(args.input_shape)
if not os.path.exists(args.output):
os.makedirs(args.output)
model_name = ('linear_' if args.isLinear else 'deep_') + args.modelType + (
'_simple' if args.simple else '')
args.output = get_output_folder(args.output + '/' + model_name, args.env)
env = gym.make(args.env)
#env = gym.wrappers.Monitor(env, args.output)
env.seed(args.seed)
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
K.set_session(sess)
K.get_session().run(tf.initialize_all_variables())
is_linear = args.isLinear
agent = DQNAgent(
q_network=create_model(4, (84, 84), env.action_space.n, is_linear,
args.modelType),
q_network2=create_model(4, (84, 84), env.action_space.n, is_linear,
args.modelType),
preprocessor=AtariPreprocessor((84, 84)),
memory=ReplayMemory(1000000, 4),
gamma=0.99,
target_update_freq=10000,
num_burn_in=50000,
train_freq=4,
batch_size=32,
is_linear=is_linear,
model_type=args.modelType,
use_replay_and_target_fixing=(not args.simple),
epsilon=0, #0.05,
action_interval=4,
output_path=args.output,
save_freq=100000)
agent.compile(lr=0.0001)
agent.fit(env, 5000000)
agent.load_weights()
agent.evaluate(env, 100, video_path_suffix='final')
env.close()
