def main(): # noqa: D103
parser = argparse.ArgumentParser(
description='Run DQN on Atari environment')
parser.add_argument('--env',
default='SpaceInvaders-v0',
help='Atari env name')
parser.add_argument('-o',
'--output',
default='atari-v0',
help='Directory to save data to')
parser.add_argument('--seed', default=0, type=int, help='Random seed')
parser.add_argument('--iters',
default=5000000,
type=int,
help='Number of interactions with environment')
parser.add_argument('--mb_size',
default=32,
type=int,
help='Minibatch size')
parser.add_argument('--max_episode_len',
default=2000,
type=int,
help='Maximum length of episode')
parser.add_argument('--frame_count',
default=4,
type=int,
help='Number of frames to feed to Q-network')
parser.add_argument('--eps',
default=0.05,
type=float,
help='Epsilon value for epsilon-greedy exploration')
parser.add_argument('--learning_rate',
default=0.0001,
type=float,
help='Learning rate for training')
parser.add_argument('--discount',
default=0.99,
type=float,
help='Discounting factor')
parser.add_argument('--replay_mem_size',
default=500000,
type=int,
help='Maximum size of replay memory')
parser.add_argument('--train_freq',
default=3,
type=int,
help='Frequency of updating Q-network')
parser.add_argument('--target_update_freq',
default=10000,
type=int,
help='Frequency of updating target network')
parser.add_argument(
'--eval',
action='store_true',
help='Indicator to evaluate model on given environment')
parser.add_argument(
'--filename',
type=str,
help='Filename for saved model to load during evaluation')
parser.add_argument(
'--model_type',
type=str,
help=
'Type of model to use: naive, linear, deep, linear_double, deep_double, dueling'
)
parser.add_argument(
'--initial_replay_size',
default=50000,
type=int,
help=
'Initial size of the replay memory upto which a uniform random policy should be used'
)
parser.add_argument('--evaluate_every',
default=5000,
type=int,
help='Number of updates to run evaluation after')
args = parser.parse_args()
#args.input_shape = tuple(args.input_shape)
# Get output folder
args.output = get_output_folder(args.output, args.env)
# Create environment
env = gym.make(args.env)
env.reset()
# Create model
preprocessed_input_shape = (84, 84)
model = create_model(args.frame_count, preprocessed_input_shape,
env.action_space.n, args.env + "-test",
args.model_type)
# Initialize replay memory
replay_mem = ReplayMemory(args.replay_mem_size, args.frame_count)
# Create agent
preprocessor_seq = PreprocessorSequence(
[AtariPreprocessor(preprocessed_input_shape)])
dqn = DQNAgent(model, preprocessor_seq, replay_mem, args.discount,
args.target_update_freq, args.initial_replay_size,
args.train_freq, args.mb_size, args.eps, args.output,
args.evaluate_every, args.model_type)
dqn.compile()
if args.eval:
dqn.eval_on_file(env, args.filename)
else:
if args.model_type == 'naive' or args.model_type == 'linear_double':
dqn.fit_naive(env, args.iters, args.max_episode_len)
else:
dqn.fit(env, args.iters, args.max_episode_len)
def main(): # noqa: D103
parser = argparse.ArgumentParser(description='Run DQN on Atari Space Invaders')
parser.add_argument('--seed', default=10703, type=int, help='Random seed')
parser.add_argument('--input_shape', default=SIZE_OF_STATE, help='Input shape')
parser.add_argument('--gamma', default=0.99, help='Discount factor')
# TODO experiment with this value.
parser.add_argument('--epsilon', default=0.1, help='Final exploration probability in epsilon-greedy')
parser.add_argument('--learning_rate', default=0.00025, help='Training learning rate.')
parser.add_argument('--batch_size', default=32, type = int, help=
'Batch size of the training part')
parser.add_argument('--question', type=int, default=7,
help='Which hw question to run.')
parser.add_argument('--evaluate', action='store_true',
help='Only affects worker. Run evaluation instead of training.')
parser.add_argument('--worker_epsilon', type=float,
help='Only affects worker. Override epsilon to use (instead of one in file).')
parser.add_argument('--skip_model_restore', action='store_true',
help='Only affects worker. Use a newly initialized model instead of restoring one.')
parser.add_argument('--generate_fixed_samples', action='store_true',
help=('Special case execution. Generate fixed samples and close. ' +
'This is necessary to run whenever the network or action space changes.'))
parser.add_argument('--ai_input_dir', default='gcloud/inputs/',
help='Input directory with initialization files.')
parser.add_argument('--ai_output_dir', default='gcloud/outputs/',
help='Output directory for gameplay files.')
parser.add_argument('--is_worker', dest='is_manager',
action='store_false',
help='Whether this is a worker (no training).')
parser.add_argument('--is_manager', dest='is_manager',
action='store_true',
help='Whether this is a manager (trains).')
parser.set_defaults(is_manager=True)
parser.add_argument('--psc', action='store_true',
help=('Only affects manager. Whether on PSC, ' +
'and should for example reduce disk usage.'))
# Copied from original phillip code (run.py).
for opt in CPU.full_opts():
opt.update_parser(parser)
parser.add_argument("--dolphin", action="store_true", default=None, help="run dolphin")
for opt in DolphinRunner.full_opts():
opt.update_parser(parser)
args = parser.parse_args()
# run.sh might pass these in via environment variable, so user directory
# might not already be expanded.
args.ai_input_dir = os.path.expanduser(args.ai_input_dir)
args.ai_output_dir = os.path.expanduser(args.ai_output_dir)
if args.is_manager:
random.seed(args.seed)
np.random.seed(args.seed)
tf.set_random_seed(args.seed)
do_evaluation = args.evaluate or random.random() < WORKER_EVALUATION_PROBABILITY
if do_evaluation or args.generate_fixed_samples:
args.cpu = EVAL_CPU_LEVEL
print('OVERRIDING cpu level to: ' + str(EVAL_CPU_LEVEL))
if args.generate_fixed_samples and args.is_manager:
raise Exception('Can not generate fixed samples as manager. Must use ' +
'--is_worker and all other necessary flags (e.g. --iso ISO_PATH)')
env = SmashEnv()
if not args.is_manager:
env.make(args) # Opens Dolphin.
question_settings = get_question_settings(args.question, args.batch_size)
online_model, online_params = create_model(
input_shape=args.input_shape,
num_actions=env.action_space.n, model_name='online_model',
create_network_fn=question_settings['create_network_fn'],
learning_rate=args.learning_rate)
target_model = online_model
update_target_params_ops = []
if (question_settings['target_update_freq'] is not None or
question_settings['is_double_network']):
target_model, target_params = create_model(
input_shape=args.input_shape,
num_actions=env.action_space.n, model_name='target_model',
create_network_fn=question_settings['create_network_fn'],
learning_rate=args.learning_rate)
update_target_params_ops = [t.assign(s) for s, t in zip(online_params, target_params)]
replay_memory = ReplayMemory(
max_size=question_settings['replay_memory_size'],
error_if_full=(not args.is_manager))
saver = tf.train.Saver(max_to_keep=None)
agent = DQNAgent(online_model=online_model,
target_model = target_model,
memory=replay_memory,
gamma=args.gamma,
target_update_freq=question_settings['target_update_freq'],
update_target_params_ops=update_target_params_ops,
batch_size=args.batch_size,
is_double_network=question_settings['is_double_network'],
is_double_dqn=question_settings['is_double_dqn'])
sess = tf.Session()
with sess.as_default():
if args.generate_fixed_samples:
print('Generating ' + str(NUM_FIXED_SAMPLES) + ' fixed samples and saving to ./' + FIXED_SAMPLES_FILENAME)
print('This file is only ever used on the manager.')
agent.compile(sess)
fix_samples = agent.prepare_fixed_samples(
env, sess, UniformRandomPolicy(env.action_space.n),
NUM_FIXED_SAMPLES, MAX_EPISODE_LENGTH)
env.terminate()
with open(FIXED_SAMPLES_FILENAME, 'wb') as f:
pickle.dump(fix_samples, f)
return
if args.is_manager or args.skip_model_restore:
agent.compile(sess)
else:
saver.restore(sess, os.path.join(args.ai_input_dir, WORKER_INPUT_MODEL_FILENAME))
print('_________________')
print('number_actions: ' + str(env.action_space.n))
# Worker code.
if not args.is_manager:
print('ai_input_dir: ' + args.ai_input_dir)
print('ai_output_dir: ' + args.ai_output_dir)
if do_evaluation:
evaluation = agent.evaluate(env, sess, GreedyPolicy(), EVAL_EPISODES, MAX_EPISODE_LENGTH)
print('Evaluation: ' + str(evaluation))
with open(FIXED_SAMPLES_FILENAME, 'rb') as fixed_samples_f:
fix_samples = pickle.load(fixed_samples_f)
mean_max_Q = calculate_mean_max_Q(sess, online_model, fix_samples)
evaluation = evaluation + (mean_max_Q,)
with open(os.path.join(args.ai_output_dir, WORKER_OUTPUT_EVALUATE_FILENAME), 'wb') as f:
pickle.dump(evaluation, f)
env.terminate()
return
worker_epsilon = args.worker_epsilon
if worker_epsilon is None:
with open(os.path.join(args.ai_input_dir, WORKER_INPUT_EPSILON_FILENAME)) as f:
lines = f.readlines()
# TODO handle unexpected lines better than just ignoring?
worker_epsilon = float(lines[0])
print('Worker epsilon: ' + str(worker_epsilon))
train_policy = GreedyEpsilonPolicy(worker_epsilon)
agent.play(env, sess, train_policy, total_seconds=PLAY_TOTAL_SECONDS, max_episode_length=MAX_EPISODE_LENGTH)
replay_memory.save_to_file(os.path.join(args.ai_output_dir, WORKER_OUTPUT_GAMEPLAY_FILENAME))
env.terminate()
return
# Manager code.
mprint('Loading fix samples')
with open(FIXED_SAMPLES_FILENAME, 'rb') as fixed_samples_f:
fix_samples = pickle.load(fixed_samples_f)
evaluation_dirs = set()
play_dirs = set()
save_model(saver, sess, args.ai_input_dir, epsilon=1.0)
epsilon_generator = LinearDecayGreedyEpsilonPolicy(
1.0, args.epsilon, TOTAL_WORKER_JOBS / 5.0)
fits_so_far = 0
mprint('Begin to train (now safe to run gcloud)')
mprint('Initial mean_max_q: ' + str(calculate_mean_max_Q(sess, online_model, fix_samples)))
while len(play_dirs) < TOTAL_WORKER_JOBS:
output_dirs = os.listdir(args.ai_output_dir)
output_dirs = [os.path.join(args.ai_output_dir, x) for x in output_dirs]
output_dirs = set(x for x in output_dirs if os.path.isdir(x))
new_dirs = sorted(output_dirs - evaluation_dirs - play_dirs)
if len(new_dirs) == 0:
time.sleep(0.1)
continue
new_dir = new_dirs[-1] # Most recent gameplay.
evaluation_path = os.path.join(new_dir, WORKER_OUTPUT_EVALUATE_FILENAME)
if os.path.isfile(evaluation_path):
evaluation_dirs.add(new_dir)
with open(evaluation_path, 'rb') as evaluation_file:
rewards, game_lengths, mean_max_Q = pickle.load(evaluation_file)
evaluation = [np.mean(rewards), np.std(rewards),
np.mean(game_lengths), np.std(game_lengths),
mean_max_Q]
mprint('Evaluation: ' + '\t'.join(str(x) for x in evaluation))
continue
memory_path = os.path.join(new_dir, WORKER_OUTPUT_GAMEPLAY_FILENAME)
try:
if os.path.getsize(memory_path) == 0:
# TODO Figure out why this happens despite temporary directory work.
# Also sometimes the file doesn't exist? Hence the try/except.
mprint('Output not ready somehow: ' + memory_path)
time.sleep(0.1)
continue
with open(memory_path, 'rb') as memory_file:
worker_memories = pickle.load(memory_file)
except Exception as exception:
print('Error reading ' + memory_path + ': ' + str(exception.args))
time.sleep(0.1)
continue
for worker_memory in worker_memories:
replay_memory.append(*worker_memory)
if args.psc:
os.remove(memory_path)
play_dirs.add(new_dir)
if len(play_dirs) <= NUM_BURN_IN_JOBS:
mprint('Skip training because still burn in.')
mprint('len(worker_memories): ' + str(len(worker_memories)))
continue
for _ in range(int(len(worker_memories) * FITS_PER_SINGLE_MEMORY)):
agent.fit(sess, fits_so_far)
fits_so_far += 1
# Partial evaluation to give frequent insight into agent progress.
# Last time checked, this took ~0.1 seconds to complete.
mprint('mean_max_q, len(worker_memories): ' +
str(calculate_mean_max_Q(sess, online_model, fix_samples)) +
', ' + str(len(worker_memories)))
# Always decrement epsilon (e.g. not just when saving model).
model_epsilon = epsilon_generator.get_epsilon(decay_epsilon=True)
if len(play_dirs) % SAVE_MODEL_EVERY == 0:
save_model(saver, sess, args.ai_input_dir, model_epsilon)
def main(): # noqa: D103
parser = argparse.ArgumentParser(description='Run DQN on Atari Breakout')
parser.add_argument('--env', default='SpaceInvaders-v0', help='Atari env name')
parser.add_argument('--network_name', default='linear_q_network', type=str, help='Type of model to use')
parser.add_argument('--window', default=4, type=int, help='how many frames are used each time')
parser.add_argument('--new_size', default=(84, 84), type=tuple, help='new size')
parser.add_argument('--batch_size', default=32, type=int, help='Batch size')
parser.add_argument('--replay_buffer_size', default=750000, type=int, help='Replay buffer size')
parser.add_argument('--gamma', default=0.99, type=float, help='Discount factor')
parser.add_argument('--alpha', default=0.0001, type=float, help='Learning rate')
parser.add_argument('--epsilon', default=0.05, type=float, help='Exploration probability for epsilon-greedy')
parser.add_argument('--target_update_freq', default=10000, type=int,
help='Frequency for copying weights to target network')
parser.add_argument('--num_burn_in', default=50000, type=int,
help='Number of prefilled samples in the replay buffer')
parser.add_argument('--num_iterations', default=5000000, type=int,
help='Number of overal interactions to the environment')
parser.add_argument('--max_episode_length', default=200000, type=int, help='Terminate earlier for one episode')
parser.add_argument('--train_freq', default=4, type=int, help='Frequency for training')
parser.add_argument('--repetition_times', default=3, type=int, help='Parameter for action repetition')
parser.add_argument('-o', '--output', default='atari-v0', type=str, help='Directory to save data to')
parser.add_argument('--seed', default=0, type=int, help='Random seed')
parser.add_argument('--experience_replay', default=False, type=bool,
help='Choose whether or not to use experience replay')
parser.add_argument('--train', default=True, type=bool, help='Train/Evaluate, set True if train the model')
parser.add_argument('--model_path', default='/media/hongbao/Study/Courses/10703/hw2/lqn_noexp',
type=str, help='specify model path to evaluation')
parser.add_argument('--max_grad', default=1.0, type=float, help='Parameter for huber loss')
parser.add_argument('--model_num', default=5000000, type=int, help='specify saved model number during train')
parser.add_argument('--log_dir', default='log', type=str, help='specify log folder to save evaluate result')
parser.add_argument('--eval_num', default=100, type=int, help='number of evaluation to run')
parser.add_argument('--save_freq', default=100000, type=int, help='model save frequency')
args = parser.parse_args()
print("\nParameters:")
for arg in vars(args):
print arg, getattr(args, arg)
print("")
env = gym.make(args.env)
num_actions = env.action_space.n
# define model object
preprocessor = AtariPreprocessor(args.new_size)
memory = ReplayMemory(args.replay_buffer_size, args.window)
# Initiating policy for both tasks (training and evaluating)
policy = LinearDecayGreedyEpsilonPolicy(args.epsilon, 0, 1000000)
if not args.train:
'''Evaluate the model'''
# check model path
if args.model_path is '':
print "Model path must be set when evaluate"
exit(1)
# specific log file to save result
log_file = os.path.join(args.log_dir, args.network_name, str(args.model_num))
model_dir = os.path.join(args.model_path, args.network_name, str(args.model_num))
with tf.Session() as sess:
# load model
with open(model_dir + ".json", 'r') as json_file:
loaded_model_json = json_file.read()
q_network_online = model_from_json(loaded_model_json)
q_network_target = model_from_json(loaded_model_json)
sess.run(tf.global_variables_initializer())
# load weights into model
q_network_online.load_weights(model_dir + ".h5")
q_network_target.load_weights(model_dir + ".h5")
dqn_agent = DQNAgent((q_network_online, q_network_target), preprocessor, memory, policy, num_actions,
args.gamma, args.target_update_freq, args.num_burn_in, args.train_freq,
args.batch_size, \
args.experience_replay, args.repetition_times, args.network_name, args.max_grad,
args.env, sess)
dqn_agent.evaluate(env, log_file, args.eval_num)
exit(0)
'''Train the model'''
q_network_online = create_model(args.window, args.new_size, num_actions, args.network_name, True)
q_network_target = create_model(args.window, args.new_size, num_actions, args.network_name, False)
# create output dir, meant to pop up error when dir exist to avoid over written
os.mkdir(os.path.join(args.output, args.network_name))
with tf.Session() as sess:
dqn_agent = DQNAgent((q_network_online, q_network_target), preprocessor, memory, policy, num_actions,
args.gamma, args.target_update_freq, args.num_burn_in, args.train_freq, args.batch_size, \
args.experience_replay, args.repetition_times, args.network_name, args.max_grad, args.env,
sess)
optimizer = tf.train.AdamOptimizer(learning_rate=args.alpha)
dqn_agent.compile(optimizer, mean_huber_loss)
dqn_agent.fit(env, args.num_iterations, os.path.join(args.output, args.network_name), args.save_freq,
args.max_episode_length)
def main(): # noqa: D103
#(SpaceInvaders-v0
# Enduro-v0
parser = argparse.ArgumentParser(description='Run DQN on Atari Breakout')
parser.add_argument('--env',
default='SpaceInvaders-v0',
help='Atari env name')
#parser.add_argument('--env', default='SpaceInvaders-v0', help='Atari env name')
#parser.add_argument('--env', default='PendulumSai-v0', help='Atari env name')
parser.add_argument('-o',
'--output',
default='atari-v0',
help='Directory to save data to')
parser.add_argument('--seed', default=0, type=int, help='Random seed')
args = parser.parse_args()
#args.input_shape = tuple(args.input_shape)
#args.output = get_output_folder(args.output, args.env)
# here is where you should start up a session,
# create your DQN agent, create your model, etc.
# then you can run your fit method.
model_name = 'Duelling_DQN'
env = gym.make(args.env)
num_iter = 2000000
max_epi_iter = 1000
epsilon = 0.4
window = 4
gamma = 0.99
target_update_freq = 10000
train_freq = 1
batch_size = 32
num_burn_in = 5000
num_actions = 3
state_size = (84, 84, 1)
new_size = state_size
max_size = 1000000
lr = 0.00025
beta_1 = 0.9
beta_2 = 0.999
epsilon2 = 1e-08
decay = 0.0
duelling_network = True
u_policy = UniformRandomPolicy(num_actions)
ge_policy = GreedyEpsilonPolicy(epsilon)
g_policy = GreedyPolicy()
policy = {
'u_policy': u_policy,
'ge_policy': ge_policy,
'g_policy': g_policy
}
#preprocessor = PreprocessorSequence([AtariPreprocessor(new_size), HistoryPreprocessor(window)])
preprocessor = AtariPreprocessor(new_size)
memory = SequentialMemory(max_size=max_size, window_length=window)
model = create_model(window, state_size, num_actions)
# print (model.summary())
dqnA = DQNAgent(q_network=model,
preprocessor=preprocessor,
memory=memory,
policy=policy,
gamma=gamma,
target_update_freq=target_update_freq,
num_burn_in=num_burn_in,
train_freq=train_freq,
batch_size=batch_size,
model_name=model_name,
duelling_network=duelling_network)
#testing
#selected_action = dqnA.select_action( np.random.rand(1,210,160,12), train=1, warmup_phase=0)
h_loss = huber_loss
optimizer = Adam(lr=lr,
beta_1=beta_1,
beta_2=beta_2,
epsilon=epsilon2,
decay=decay)
dqnA.compile(optimizer, h_loss)
#callback1 = ProgbarLogger(count_mode='samples')
dqnA.fit(env, num_iterations=num_iter, max_episode_length=max_epi_iter)
def main(): # noqa: D103
parser = argparse.ArgumentParser(description='Run DQN on Atari Breakout')
parser.add_argument('--env', default='Breakout-v0', help='Atari env name')
parser.add_argument(
'-o', '--output', default='atari-v0', help='Directory to save data to')
parser.add_argument('--seed', default=0, type=int, help='Random seed')
parser.add_argument('-ni', '--num_iterations', default=10, type=int, help='Num of iterations for training')
parser.add_argument('-m', '--max_episode_length', default=60, type=int, help='Max episode length of a sequence')
parser.add_argument('-ne', '--num_episodes', default=10, type=int, help='Num of epsidoes for evaluating')
parser.add_argument('-r', '--replay_memory', default=10, type=int, help='The size of replay memory')
parser.add_argument('-gamma', '--discount_factor', default=0.99, type=float, help='Discount factor of MDP')
parser.add_argument('-ge', '--Greedy_epsilon', default=0.95, type=float, help='The probability to choose a greedy action')
args = parser.parse_args()
#args.input_shape = tuple(args.input_shape)
args.output = get_output_folder(args.output, args.env)
# the dirs to store results
os.makedirs(args.output)
os.chdir(args.output)
# here is where you should start up a session,
# create your DQN agent, create your model, etc.
# then you can run your fit method.
env = gym.make('Breakout-v0')
env.reset()
# Preprocess image
preprocess_network = preprocessors.PreprocessorSequence('network')
preprocess_memory = preprocessors.PreprocessorSequence('memory')
# Policy choose
Greedy = policy.GreedyEpsilonPolicy(0.95)
DG = policy.LinearDecayGreedyEpsilonPolicy('attr_name', 1, 0.1, 1000000)
# Create model from Atari paper
model = create_model(window=4, input_shape=(84, 84), num_actions=4)
# load weights
location = '/'
# Define tensorboard
tensorboard = keras.callbacks.TensorBoard(log_dir='./logs', histogram_freq=0, write_graph=True, write_images=True)
# Optimazor
optimizor = Adam(lr=0.00025)
# Create memory
memory = core.ReplayMemory(max_size=args.replay_memory, phi_length=4, window_height=84, window_length=84, rng=np.random.RandomState(100))
agent = DQNAgent(q_network=model, target=model, preprocessor={'network': preprocess_network, 'memory': preprocess_memory},
memory=memory, policy={'Greedy': Greedy, 'DG': DG}, gamma=args.discount_factor, target_update_freq=100000, num_burn_in=args.replay_memory, train_freq=4, batch_size=32
,callbacks=tensorboard)
agent.compile(optimizer= optimizor, loss_func=objectives.mean_huber_loss)
agent.init_memory(env=env, max_episode_length=30)
agent.fit(env=env, num_iterations=args.num_iterations, max_episode_length=args.max_episode_length)
agent.evaluate(env=env, num_episodes=args.num_episodes, max_episode_length=args.max_episode_length)
# store the hyperameters
file_abs = "./hypermeters"
with open(file_abs, "w") as f:
f.write("Num of iterations:")
f.write(str(args.num_iterations) + '\n')
f.write("Max epsidoe length:")
f.write(str(args.max_episode_length) + '\n')
f.write("Num of episodes:")
f.write(str(args.num_episodes) + '\n')
f.write("Replay memory:")
f.write(str(args.replay_memory) + '\n')
f.write("Discount factor:")
f.write(str(args.discount_factor) + '\n')
def main():
parser = argparse.ArgumentParser(description='Run DQN on Atari Breakout')
parser.add_argument('--env', default='Breakout-v0', help='Atari env name')
parser.add_argument('-o',
'--output',
default='atari-v0',
help='Directory to save data to')
parser.add_argument('--seed', default=0, type=int, help='Random seed')
parser.add_argument('--mode', choices=['train', 'test'], default='test')
parser.add_argument('--network',
choices=['deep', 'linear'],
default='deep')
parser.add_argument('--method',
choices=['dqn', 'double', 'dueling'],
default='dqn')
parser.add_argument('--monitor', type=bool, default=True)
parser.add_argument('--iter', type=int, default=2400000)
parser.add_argument('--test_policy',
choices=['Greedy', 'GreedyEpsilon'],
default='GreedyEpsilon')
args = parser.parse_args()
args.seed = np.random.randint(0, 1000000, 1)[0]
args.weights = 'models/dqn_{}_weights_{}_{}_{}.h5f'.format(
args.env, args.method, args.network, args.iter)
args.monitor_path = 'tmp/dqn_{}_weights_{}_{}_{}_{}'.format(
args.env, args.method, args.network, args.iter, args.test_policy)
if args.mode == 'train':
args.monitor = False
env = gym.make(args.env)
if args.monitor:
env = wrappers.Monitor(env, args.monitor_path)
np.random.seed(args.seed)
env.seed(args.seed)
args.gamma = 0.99
args.learning_rate = 0.0001
args.epsilon = 0.05
args.num_iterations = 5000000
args.batch_size = 32
args.window_length = 4
args.num_burn_in = 50000
args.target_update_freq = 10000
args.log_interval = 10000
args.model_checkpoint_interval = 10000
args.train_freq = 4
args.num_actions = env.action_space.n
args.input_shape = (84, 84)
args.memory_max_size = 1000000
args.output = get_output_folder(args.output, args.env)
args.suffix = args.method + '_' + args.network
if (args.method == 'dqn'):
args.enable_double_dqn = False
args.enable_dueling_network = False
elif (args.method == 'double'):
args.enable_double_dqn = True
args.enable_dueling_network = False
elif (args.method == 'dueling'):
args.enable_double_dqn = False
args.enable_dueling_network = True
else:
print('Attention! Method Worng!!!')
if args.test_policy == 'Greedy':
test_policy = GreedyPolicy()
elif args.test_policy == 'GreedyEpsilon':
test_policy = GreedyEpsilonPolicy(args.epsilon)
print(args)
K.tensorflow_backend.set_session(get_session())
model = create_model(args.window_length, args.input_shape,
args.num_actions, args.network)
# we create our preprocessor, the Ataripreprocessor will only process current frame the agent is seeing. And the sequence
# preprocessor will construct the state by concatenating 3 previous frames from HistoryPreprocessor and current processed frame
Processor = {}
Processor['Atari'] = AtariPreprocessor(args.input_shape)
Processor['History'] = HistoryPreprocessor(args.window_length)
ProcessorSequence = PreprocessorSequence(Processor) # construct 84x84x4
# we create our memory for saving all experience collected during training with window length 4
memory = ReplayMemory(max_size=args.memory_max_size,
input_shape=args.input_shape,
window_length=args.window_length)
# we use linear decay greedy epsilon policy and tune the epsilon from 1 to 0.1 during the first 100w iterations and then keep using
# epsilon with 0.1 to further train the network
policy = LinearDecayGreedyEpsilonPolicy(GreedyEpsilonPolicy(args.epsilon),
attr_name='eps',
start_value=1,
end_value=0.1,
num_steps=1000000)
# we construct our agent and use 0.99 as our discounted factor, 32 as our batch_size. We update our model for each 4 iterations. But during first
# 50000 iterations, we only collect data to the memory and don't update our model.
dqn = DQNAgent(q_network=model,
policy=policy,
memory=memory,
num_actions=args.num_actions,
test_policy=test_policy,
preprocessor=ProcessorSequence,
gamma=args.gamma,
target_update_freq=args.target_update_freq,
num_burn_in=args.num_burn_in,
train_freq=args.train_freq,
batch_size=args.batch_size,
enable_double_dqn=args.enable_double_dqn,
enable_dueling_network=args.enable_dueling_network)
adam = Adam(lr=args.learning_rate)
dqn.compile(optimizer=adam)
if args.mode == 'train':
weights_filename = 'dqn_{}_weights_{}.h5f'.format(
args.env, args.suffix)
checkpoint_weights_filename = 'dqn_' + args.env + '_weights_' + args.suffix + '_{step}.h5f'
log_filename = 'dqn_{}_log_{}.json'.format(args.env, args.suffix)
log_dir = '../tensorboard_{}_log_{}'.format(args.env, args.suffix)
callbacks = [
ModelIntervalCheckpoint(checkpoint_weights_filename,
interval=args.model_checkpoint_interval)
]
callbacks += [FileLogger(log_filename, interval=100)]
callbacks += [
TensorboardStepVisualization(log_dir=log_dir,
histogram_freq=1,
write_graph=True,
write_images=True)
]
# start training
# we don't apply action repetition explicitly since the game will randomly skip frame itself
dqn.fit(env,
callbacks=callbacks,
verbose=1,
num_iterations=args.num_iterations,
action_repetition=1,
log_interval=args.log_interval,
visualize=True)
dqn.save_weights(weights_filename, overwrite=True)
dqn.evaluate(env,
num_episodes=10,
visualize=True,
num_burn_in=5,
action_repetition=1)
elif args.mode == 'test':
weights_filename = 'dqn_{}_weights_{}.h5f'.format(
args.env, args.suffix)
if args.weights:
weights_filename = args.weights
dqn.load_weights(weights_filename)
dqn.evaluate(env,
num_episodes=250,
visualize=True,
num_burn_in=5,
action_repetition=1)
# we upload our result to openai gym
if args.monitor:
env.close()
gym.upload(args.monitor_path, api_key='sk_J62obX9PQg2ExrM6H9rvzQ')
def main(): # noqa: D103
parser = argparse.ArgumentParser(description='Run DQN on Atari Breakout')
parser.add_argument('--env',
default='SpaceInvadersDeterministic-v3',
help='Atari env name')
parser.add_argument('-o',
'--output',
default='atari-v0',
help='Directory to save data to')
parser.add_argument('--seed', default=0, type=int, help='Random seed')
parser.add_argument('--model',
default='dqn',
help='Q Network type to use.')
parser.add_argument('--double', action='store_true')
model_map = {
'linear': LinearQN,
'mlp': MLP,
'dqn': DQN,
'dueling': DuelingDQN
}
args = parser.parse_args()
args.model = args.model.lower()
if args.model not in model_map:
print("Invalid model type. Valid types are", model_map.keys())
sys.exit(1)
args.output = get_output_folder(args.output, args.env)
# here is where you should start up a session,
# create your DQN agent, create your model, etc.
# then you can run your fit method.
env = gym.make(args.env)
monitored_env = gym.wrappers.Monitor(
gym.make(args.env),
args.output,
video_callable=lambda i: i % EVAL_NUM_EPISODES == 0)
atari = not args.env.startswith("CartPole")
if atari:
input_shape = (IMAGE_SIZE, IMAGE_SIZE)
preprocessor = lambda: PreprocessorSequence(
AtariPreprocessor(new_size=input_shape),
HistoryPreprocessor(history_length=WINDOW_SIZE, max_over=True))
else:
input_shape = (4, )
preprocessor = lambda: HistoryPreprocessor(history_length=WINDOW_SIZE)
memory = ExperienceReplay(max_size=REPLAY_BUFFER_SIZE,
window_length=WINDOW_SIZE)
NUM_ACTIONS = env.action_space.n
#policy = UniformRandomPolicy(num_actions=NUM_ACTIONS)
#policy = GreedyEpsilonPolicy(NUM_ACTIONS, EPSILON)
policy = LinearDecayGreedyEpsilonPolicy(NUM_ACTIONS, 1.0, EPSILON,
NUM_ITERATIONS_LINEAR_DECAY)
model = model_map[args.model](exp_name=args.output)
agent = DQNAgent(q_network=model,
preprocessor=preprocessor,
memory=memory,
policy=policy,
gamma=GAMMA,
target_update_freq=TARGET_UPDATE_FREQ,
replay_buffer_size=REPLAY_BUFFER_SIZE,
train_freq=TRAIN_FREQ,
batch_size=BATCH_SIZE,
output_dir=args.output,
double_dqn=args.double)
agent.compile(window=WINDOW_SIZE,
input_shape=input_shape,
num_actions=NUM_ACTIONS,
model_name='q_network')
signal.signal(signal.SIGINT, agent.signal_handler)
signal.signal(signal.SIGTERM, agent.signal_handler)
signal.signal(signal.SIGHUP, agent.signal_handler)
agent.fit(env, monitored_env, num_iterations=NUM_ITERATIONS)
def testPerformance(self):
"""
Test to make sure each model(DQN, DDQN, DoubleQN) could be created and compiled
"""
#create a model of the world
env = gym.make("SpaceInvaders-v0")
env.frameskip = 1
#create a fake keras model
input_shape = (84, 84)
window = 4
num_actions = env.action_space.n
model = Sequential(name="test_model")
model.add(
Convolution2D(filters=16,
kernel_size=8,
strides=4,
activation='relu',
input_shape=(input_shape[0], input_shape[1],
window)))
model.add(
Convolution2D(filters=32,
kernel_size=4,
strides=2,
activation='relu'))
model.add(
Convolution2D(filters=64,
kernel_size=3,
strides=1,
activation='relu'))
model.add(Flatten())
model.add(Dense(units=512, activation='relu'))
model.add(Dense(units=num_actions, activation='linear'))
#create loss function & optimizer
optimizer = Adam(lr=0.001,
beta_1=0.9,
beta_2=0.999,
epsilon=1e-08,
decay=0.0)
loss_func = huber_loss
#preprocessors
history_prep = HistoryPreprocessor(4)
atari_prep = AtariPreprocessor(input_shape, 0, 999)
numpy_prep = NumpyPreprocessor()
preprocessors = PreprocessorSequence(
[atari_prep, history_prep, numpy_prep]) #from left to right
memory = ActionReplayMemory(100000, 4)
#policy = LinearDecayGreedyEpsilonPolicy(1, 0.1,100000)
policy = SamePolicy(1)
#agent = DQNAgent(model, preprocessors, memory, policy,0.99, target_update_freq,None,train_freq,batch_size)
dqn_agent = DQNAgent(model, preprocessors, memory, policy, 0.99, 10000,
None, 4, 32)
dqn_agent.compile(optimizer, loss_func)
total_time = 0
times = 50
for i in range(0, times):
start_time = time.time()
dqn_agent.evaluate_detailed(env, 1)
total_time += (time.time() - start_time)
sys.stdout.write('\r{}'.format(i))
sys.stdout.flush()
print("average evaluation time:{} total time:{}".format(
total_time / times, total_time))
def main(): # noqa: D103
parser = argparse.ArgumentParser(
description='Run DQN on given game environment')
parser.add_argument('--env',
default='SpaceInvaders-v0',
help='Atari env name')
parser.add_argument('-o',
'--output',
default='train',
help='Directory to save data to')
parser.add_argument('--seed', default=0, type=int, help='Random seed')
parser.add_argument('--gamma',
default=0.99,
type=float,
help='Discount factor')
parser.add_argument(
'--target_update_freq',
default=10000,
type=int,
help='interval between two updates of the target network')
parser.add_argument(
'--num_burn_in',
default=10000,
type=int,
help=
'number of samples to be filled into the replay memory before updating the network'
)
parser.add_argument('--train_freq',
default=1,
type=int,
help='How often to update the Q-network')
parser.add_argument('--batch_size',
default=32,
type=int,
help='batch_size')
parser.add_argument('--num_iterations',
default=100000,
type=int,
help="num of iterations to run for the training")
parser.add_argument('--max_episode_length',
default=10000,
type=int,
help='max length of one episode')
parser.add_argument('--lr',
default=0.0001,
type=float,
help='learning rate')
parser.add_argument('--epsilon',
default=0.05,
type=float,
help='epsilon for exploration')
parser.add_argument('--experiment_id',
default=None,
type=int,
help='index of experiment to reload checkpoint')
parser.add_argument('--save_freq',
default=10000,
type=int,
help='checkpoint saving frequency')
parser.add_argument(
'--evaluate_freq',
default=10000,
type=int,
help='frequency to do evaluation and record video by wrapper')
parser.add_argument('--test_num_episodes',
default=10,
type=int,
help='number of episodes to play at each evaluation')
args = parser.parse_args()
if not args.experiment_id:
args.output = get_output_folder(args.output, args.env)
else:
args.output = os.path.join(args.output, args.env) + '-run{}'.format(
args.experiment_id)
# print(args.output)
# if not os.path.exists('./'+args.output):
# print("create folder")
# print(args.output)
# os.mkdir(args.output)
game_env = gym.make(args.env)
num_actions = game_env.action_space.n
input_shape = (84, 84)
#todo: setup logger
#writer = tf.summary.FileWriter()
#setup model
model = create_model(window=4,
input_shape=input_shape,
num_actions=num_actions,
model_name='linear_model')
#setup optimizer
#optimizer = Adam(lr=args.lr)
optimizer = tf.train.AdamOptimizer(learning_rate=args.lr)
#setup preprocessor
atari_preprocessor = AtariPreprocessor(input_shape)
history_preprocessor = HistoryPreprocessor(history_length=3)
preprocessor = PreprocessorSequence(
[atari_preprocessor, history_preprocessor])
#setup policy
policy = GreedyEpsilonPolicy(epsilon=args.epsilon, num_actions=num_actions)
#setup DQN agent
agent = DQNAgent(q_network=model,
preprocessor=preprocessor,
memory=None,
policy=policy,
gamma=args.gamma,
target_update_freq=args.target_update_freq,
num_burn_in=args.num_burn_in,
train_freq=args.train_freq,
batch_size=args.batch_size,
logdir=args.output,
save_freq=args.save_freq,
evaluate_freq=args.evaluate_freq,
test_num_episodes=args.test_num_episodes)
agent.compile(optimizer=optimizer, loss_func=mean_huber_loss)
agent.fit(env=game_env,
num_iterations=args.num_iterations,
max_episode_length=args.max_episode_length)
def main(): # noqa: D103
parser = argparse.ArgumentParser(description='Run DQN on Atari Breakout')
#parser.add_argument('--env', default='Breakout-v0', help='Atari env name')
parser.add_argument('--env',
default='SpaceInvaders-v0',
help='Atari env name')
parser.add_argument('--output',
default='results',
help='Directory to save data to')
parser.add_argument('-l',
'--isLinear',
default=0,
type=int,
choices=range(0, 2),
help='1: use linear model; 0: use deep model')
parser.add_argument(
'-m',
'--modelType',
default='q',
choices=['q', 'double', 'dueling'],
help=
'q: q learning; double: double q learning; dueling: dueling q learning'
)
parser.add_argument(
'-s',
'--simple',
default=0,
type=int,
choices=range(0, 2),
help=
'1: without replay or target fixing ; 0: use replay and target fixing')
parser.add_argument('--seed', default=0, type=int, help='Random seed')
args = parser.parse_args()
#args.input_shape = tuple(args.input_shape)
if not os.path.exists(args.output):
os.makedirs(args.output)
model_name = ('linear_' if args.isLinear else 'deep_') + args.modelType + (
'_simple' if args.simple else '')
args.output = get_output_folder(args.output + '/' + model_name, args.env)
env = gym.make(args.env)
#env = gym.wrappers.Monitor(env, args.output)
env.seed(args.seed)
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
K.set_session(sess)
K.get_session().run(tf.initialize_all_variables())
is_linear = args.isLinear
agent = DQNAgent(
q_network=create_model(4, (84, 84), env.action_space.n, is_linear,
args.modelType),
q_network2=create_model(4, (84, 84), env.action_space.n, is_linear,
args.modelType),
preprocessor=AtariPreprocessor((84, 84)),
memory=ReplayMemory(1000000, 4),
gamma=0.99,
target_update_freq=10000,
num_burn_in=50000,
train_freq=4,
batch_size=32,
is_linear=is_linear,
model_type=args.modelType,
use_replay_and_target_fixing=(not args.simple),
epsilon=0, #0.05,
action_interval=4,
output_path=args.output,
save_freq=100000)
agent.compile(lr=0.0001)
agent.fit(env, 5000000)
agent.load_weights()
agent.evaluate(env, 100, video_path_suffix='final')
env.close()
def main(): # noqa: D103
parser = argparse.ArgumentParser(
description='Run DQN on given game environment')
parser.add_argument('--env',
default='SpaceInvaders-v0',
help='Atari env name')
parser.add_argument('-o',
'--output',
default='SpaceInvaders-v0',
help='Directory to save data to')
parser.add_argument('--seed', default=0, type=int, help='Random seed')
parser.add_argument('--gamma',
default=0.99,
type=float,
help='Discount factor')
parser.add_argument(
'--target_update_freq',
default=10000,
type=int,
help='interval between two updates of the target network')
parser.add_argument(
'--num_burn_in',
default=10,
type=int,
help=
'number of samples to be filled into the replay memory before updating the network'
)
parser.add_argument('--train_freq',
default=1,
type=int,
help='How often to update the Q-network')
parser.add_argument('--batch_size',
default=32,
type=int,
help='batch_size')
parser.add_argument('--num_iterations',
default=10000,
type=int,
help="num of iterations to run for the training")
parser.add_argument('--max_episode_length',
default=10000,
type=int,
help='max length of one episode')
parser.add_argument('--lr',
default=0.0001,
type=float,
help='learning rate')
parser.add_argument('--epsilon',
default=0.05,
type=float,
help='epsilon for exploration')
args = parser.parse_args()
args.output = get_output_folder(args.output, args.env)
game_env = gym.make(args.env)
num_actions = game_env.action_space.n
input_shape = (84, 84)
#setup logger
writer = tf.summary.FileWriter()
#setup model
model = create_model(window=4,
input_shape=input_shape,
num_actions=num_actions,
model_name='linear_model')
#setup optimizer
optimizer = Adam(lr=args.lr)
#setup preprocessor
atari_preprocessor = AtariPreprocessor(input_shape)
history_preprocessor = HistoryPreprocessor(history_length=3)
preprocessor = PreprocessorSequence(
[atari_preprocessor, history_preprocessor])
#setup policy
policy = GreedyEpsilonPolicy(epsilon=args.epsilon, num_actions=num_actions)
#setup DQN agent
agent = DQNAgent(q_network=model,
preprocessor=preprocessor,
memory=None,
policy=policy,
gamma=args.gamma,
target_update_freq=args.target_update_freq,
num_burn_in=args.num_burn_in,
train_freq=args.train_freq,
batch_size=args.batch_size)
agent.compile(optimizer=optimizer, loss_func=mean_huber_loss)
agent.fit(env=game_env,
num_iterations=args.num_iterations,
max_episode_length=args.max_episode_length)
