def main(): # noqa: D103
parser = argparse.ArgumentParser(description='Run DQN on Atari Breakout')
parser.add_argument('--env', default='Breakout-v0', help='Atari env name')
parser.add_argument('-o',
'--output',
default='atari-v0',
help='Directory to save data to')
parser.add_argument('--seed', default=0, type=int, help='Random seed')
args = parser.parse_args()
#args.input_shape = tuple(args.input_shape)
#args.output = get_output_folder(args.output, args.env)
#set up environment model
env = gym.make(str(args.env))
NUM_ACTIONS = env.action_space.n #env.get_action_space().num_actions()
#make dqn agent
FRAMES_PER_STATE = 4
INPUT_SHAPE = (84, 84)
GAMMA = .99
NUM_ITERATIONS = 10000
TARGET_UPDATE_FREQ = 1000
NUM_BURN_IN = 1000
TRAIN_FREQ = 0
BATCH_SIZE = 32
REPLAY_MEM_SIZE = 1000
REPLAY_START_SIZE = 1000
MAX_EPISODE_LEN = 1000
HELD_OUT_STATES_SIZE = 1000
model = create_model(FRAMES_PER_STATE,
INPUT_SHAPE,
NUM_ACTIONS,
model_name='linear q_network')
plot_model(model, to_file='model.png')
target = create_model(FRAMES_PER_STATE,
INPUT_SHAPE,
NUM_ACTIONS,
model_name='linear q_network target')
preprocessor = HistoryPreprocessor(FRAMES_PER_STATE - 1)
memory = ReplayMemory(REPLAY_MEM_SIZE, FRAMES_PER_STATE)
held_out_states = ReplayMemory(HELD_OUT_STATES_SIZE, FRAMES_PER_STATE)
policy = LinearDecayGreedyEpsilonPolicy(1, .05, int(1e6))
agent = DQNAgent(model, target, preprocessor, memory, policy,
held_out_states, HELD_OUT_STATES_SIZE, GAMMA,
TARGET_UPDATE_FREQ, NUM_BURN_IN, TRAIN_FREQ, BATCH_SIZE,
REPLAY_START_SIZE, NUM_ACTIONS)
#compile agent
adam = Adam(lr=0.0001)
loss = mean_huber_loss
agent.compile(adam, loss)
agent.fit(env, NUM_ITERATIONS, MAX_EPISODE_LEN)
model_json = model.to_json()
with open("model.json", "w") as json_file:
json_file.write(model_json)
# serialize weights to HDF5
model.save_weights("model.h5")
print("Saved model to disk")
def main():
#env = gym.make("Enduro-v0")
#env = gym.make("SpaceInvaders-v0")
#env = gym.make("Breakout-v0")
model_name = "q2"
if (len(sys.argv) >= 2):
model_name = sys.argv[1]
if (len(sys.argv) >= 3):
env = gym.make(sys.argv[2])
else:
#env = gym.make("Enduro-v0")
env = gym.make("SpaceInvaders-v0")
#env = gym.make("Breakout-v0")
#no skip frames
env.frameskip = 1
input_shape = (84, 84)
batch_size = 1
num_actions = env.action_space.n
memory_size = 2 #2 because it need to save the current state and the future state, no matter what it gets, it will always just pick the earlier one
memory_burn_in_num = 1
start_epsilon = 1
end_epsilon = 0.01
decay_steps = 1000000
target_update_freq = 1 #no targeting
train_freq = 4 #How often you train the network
history_size = 4
history_prep = HistoryPreprocessor(history_size)
atari_prep = AtariPreprocessor(input_shape, 0, 999)
numpy_prep = NumpyPreprocessor()
preprocessors = PreprocessorSequence(
[atari_prep, history_prep, numpy_prep]) #from left to right
policy = LinearDecayGreedyEpsilonPolicy(start_epsilon, end_epsilon,
decay_steps)
linear_model = create_model(history_size, input_shape, num_actions,
model_name)
optimizer = Adam(lr=0.001,
beta_1=0.9,
beta_2=0.999,
epsilon=1e-08,
decay=0.0)
loss_func = huber_loss
#linear_model.compile(optimizer, loss_func)
linear_model.summary()
random_policy = UniformRandomPolicy(num_actions)
#memory = ActionReplayMemory(1000000,4)
memory = ActionReplayMemory(memory_size, history_size)
#memory_burn_in(env,memory,preprocessors,memory_burn_in_num,random_policy)
#print(reward_arr)
#print(curr_state_arr)
agent = DQNAgent(linear_model, preprocessors, memory, policy, 0.99,
target_update_freq, None, train_freq, batch_size)
agent.compile(optimizer, loss_func)
agent.save_models()
agent.fit(env, 1000000, 100000)
def main():
parser = argparse.ArgumentParser(description='Run DQN on Atari Breakout')
parser.add_argument('--env', default='Breakout-v0', help='Atari env name')
parser.add_argument('-o',
'--output',
default='atari-v0',
help='Directory to save data to')
parser.add_argument('--seed', default=0, type=int, help='Random seed')
parser.add_argument('--type',
default="DQN",
help='Type of network to train. ()')
args = parser.parse_args()
#check if valid network type
network_types = [
"Linear", "LinearERTF", "DoubleLinear", "DQN", "DDQN", "Duling"
]
if (not (args.type in network_types)):
raise ValueError("Invalid network type.")
NETWORK_TYPE = args.type
#set up environment model
env = gym.make(str(args.env))
NUM_ACTIONS = env.action_space.n
#make dqn agent
"""
FRAMES_PER_STATE = 4
INPUT_SHAPE = (84,84)
GAMMA = .99
NUM_ITERATIONS = 1000000
TARGET_UPDATE_FREQ = 100000
BATCH_SIZE = 32
REPLAY_MEM_SIZE = 1000000
REPLAY_START_SIZE = 50000
MAX_EPISODE_LEN = 100
REWARD_SAMPLE = 1000
HELD_OUT_STATES_SIZE=1000
"""
FRAMES_PER_STATE = 4
INPUT_SHAPE = (84, 84)
GAMMA = .99
NUM_ITERATIONS = 20000
TARGET_UPDATE_FREQ = 1000
BATCH_SIZE = 32
REPLAY_MEM_SIZE = 1000000
REPLAY_START_SIZE = 1000
MAX_EPISODE_LEN = 10
REWARD_SAMPLE = 1000
HELD_OUT_STATES_SIZE = 1000
#retuns a list of models ie: [Online,None] or [Online,Target] or [OnlineA,OnlineB]
models = create_model(FRAMES_PER_STATE, INPUT_SHAPE, NUM_ACTIONS,
NETWORK_TYPE)
history = HistoryPreprocessor(FRAMES_PER_STATE - 1)
preprocessor = Preprocessor()
if (NETWORK_TYPE != "Linear"):
memory = ReplayMemory(REPLAY_MEM_SIZE, FRAMES_PER_STATE)
else:
memory = None
held_out_states = ReplayMemory(HELD_OUT_STATES_SIZE, FRAMES_PER_STATE)
policy = LinearDecayGreedyEpsilonPolicy(1, .05, int(1e6))
agent = DQNAgent(models[0], models[1], preprocessor, history, memory,
policy, GAMMA, TARGET_UPDATE_FREQ, BATCH_SIZE,
REPLAY_START_SIZE, NUM_ACTIONS, NETWORK_TYPE,
REWARD_SAMPLE, held_out_states, HELD_OUT_STATES_SIZE)
#compile agent
adam = Adam(lr=0.0001)
loss = mean_huber_loss
agent.compile(adam, loss)
agent.fit(env, NUM_ITERATIONS, MAX_EPISODE_LEN)
model_json = models[0].to_json()
with open(NETWORK_TYPE + "model.json", "w") as json_file:
json_file.write(model_json)
# serialize weights to HDF5
models[0].save_weights(NETWORK_TYPE + "model.h5")
print("Saved model to disk")
def main(): # noqa: D103
parser = argparse.ArgumentParser(description='Run DQN on Atari Space Invaders')
parser.add_argument('--seed', default=10703, type=int, help='Random seed')
parser.add_argument('--input_shape', default=SIZE_OF_STATE, help='Input shape')
parser.add_argument('--gamma', default=0.99, help='Discount factor')
# TODO experiment with this value.
parser.add_argument('--epsilon', default=0.1, help='Final exploration probability in epsilon-greedy')
parser.add_argument('--learning_rate', default=0.00025, help='Training learning rate.')
parser.add_argument('--batch_size', default=32, type = int, help=
'Batch size of the training part')
parser.add_argument('--question', type=int, default=7,
help='Which hw question to run.')
parser.add_argument('--evaluate', action='store_true',
help='Only affects worker. Run evaluation instead of training.')
parser.add_argument('--worker_epsilon', type=float,
help='Only affects worker. Override epsilon to use (instead of one in file).')
parser.add_argument('--skip_model_restore', action='store_true',
help='Only affects worker. Use a newly initialized model instead of restoring one.')
parser.add_argument('--generate_fixed_samples', action='store_true',
help=('Special case execution. Generate fixed samples and close. ' +
'This is necessary to run whenever the network or action space changes.'))
parser.add_argument('--ai_input_dir', default='gcloud/inputs/',
help='Input directory with initialization files.')
parser.add_argument('--ai_output_dir', default='gcloud/outputs/',
help='Output directory for gameplay files.')
parser.add_argument('--is_worker', dest='is_manager',
action='store_false',
help='Whether this is a worker (no training).')
parser.add_argument('--is_manager', dest='is_manager',
action='store_true',
help='Whether this is a manager (trains).')
parser.set_defaults(is_manager=True)
parser.add_argument('--psc', action='store_true',
help=('Only affects manager. Whether on PSC, ' +
'and should for example reduce disk usage.'))
# Copied from original phillip code (run.py).
for opt in CPU.full_opts():
opt.update_parser(parser)
parser.add_argument("--dolphin", action="store_true", default=None, help="run dolphin")
for opt in DolphinRunner.full_opts():
opt.update_parser(parser)
args = parser.parse_args()
# run.sh might pass these in via environment variable, so user directory
# might not already be expanded.
args.ai_input_dir = os.path.expanduser(args.ai_input_dir)
args.ai_output_dir = os.path.expanduser(args.ai_output_dir)
if args.is_manager:
random.seed(args.seed)
np.random.seed(args.seed)
tf.set_random_seed(args.seed)
do_evaluation = args.evaluate or random.random() < WORKER_EVALUATION_PROBABILITY
if do_evaluation or args.generate_fixed_samples:
args.cpu = EVAL_CPU_LEVEL
print('OVERRIDING cpu level to: ' + str(EVAL_CPU_LEVEL))
if args.generate_fixed_samples and args.is_manager:
raise Exception('Can not generate fixed samples as manager. Must use ' +
'--is_worker and all other necessary flags (e.g. --iso ISO_PATH)')
env = SmashEnv()
if not args.is_manager:
env.make(args) # Opens Dolphin.
question_settings = get_question_settings(args.question, args.batch_size)
online_model, online_params = create_model(
input_shape=args.input_shape,
num_actions=env.action_space.n, model_name='online_model',
create_network_fn=question_settings['create_network_fn'],
learning_rate=args.learning_rate)
target_model = online_model
update_target_params_ops = []
if (question_settings['target_update_freq'] is not None or
question_settings['is_double_network']):
target_model, target_params = create_model(
input_shape=args.input_shape,
num_actions=env.action_space.n, model_name='target_model',
create_network_fn=question_settings['create_network_fn'],
learning_rate=args.learning_rate)
update_target_params_ops = [t.assign(s) for s, t in zip(online_params, target_params)]
replay_memory = ReplayMemory(
max_size=question_settings['replay_memory_size'],
error_if_full=(not args.is_manager))
saver = tf.train.Saver(max_to_keep=None)
agent = DQNAgent(online_model=online_model,
target_model = target_model,
memory=replay_memory,
gamma=args.gamma,
target_update_freq=question_settings['target_update_freq'],
update_target_params_ops=update_target_params_ops,
batch_size=args.batch_size,
is_double_network=question_settings['is_double_network'],
is_double_dqn=question_settings['is_double_dqn'])
sess = tf.Session()
with sess.as_default():
if args.generate_fixed_samples:
print('Generating ' + str(NUM_FIXED_SAMPLES) + ' fixed samples and saving to ./' + FIXED_SAMPLES_FILENAME)
print('This file is only ever used on the manager.')
agent.compile(sess)
fix_samples = agent.prepare_fixed_samples(
env, sess, UniformRandomPolicy(env.action_space.n),
NUM_FIXED_SAMPLES, MAX_EPISODE_LENGTH)
env.terminate()
with open(FIXED_SAMPLES_FILENAME, 'wb') as f:
pickle.dump(fix_samples, f)
return
if args.is_manager or args.skip_model_restore:
agent.compile(sess)
else:
saver.restore(sess, os.path.join(args.ai_input_dir, WORKER_INPUT_MODEL_FILENAME))
print('_________________')
print('number_actions: ' + str(env.action_space.n))
# Worker code.
if not args.is_manager:
print('ai_input_dir: ' + args.ai_input_dir)
print('ai_output_dir: ' + args.ai_output_dir)
if do_evaluation:
evaluation = agent.evaluate(env, sess, GreedyPolicy(), EVAL_EPISODES, MAX_EPISODE_LENGTH)
print('Evaluation: ' + str(evaluation))
with open(FIXED_SAMPLES_FILENAME, 'rb') as fixed_samples_f:
fix_samples = pickle.load(fixed_samples_f)
mean_max_Q = calculate_mean_max_Q(sess, online_model, fix_samples)
evaluation = evaluation + (mean_max_Q,)
with open(os.path.join(args.ai_output_dir, WORKER_OUTPUT_EVALUATE_FILENAME), 'wb') as f:
pickle.dump(evaluation, f)
env.terminate()
return
worker_epsilon = args.worker_epsilon
if worker_epsilon is None:
with open(os.path.join(args.ai_input_dir, WORKER_INPUT_EPSILON_FILENAME)) as f:
lines = f.readlines()
# TODO handle unexpected lines better than just ignoring?
worker_epsilon = float(lines[0])
print('Worker epsilon: ' + str(worker_epsilon))
train_policy = GreedyEpsilonPolicy(worker_epsilon)
agent.play(env, sess, train_policy, total_seconds=PLAY_TOTAL_SECONDS, max_episode_length=MAX_EPISODE_LENGTH)
replay_memory.save_to_file(os.path.join(args.ai_output_dir, WORKER_OUTPUT_GAMEPLAY_FILENAME))
env.terminate()
return
# Manager code.
mprint('Loading fix samples')
with open(FIXED_SAMPLES_FILENAME, 'rb') as fixed_samples_f:
fix_samples = pickle.load(fixed_samples_f)
evaluation_dirs = set()
play_dirs = set()
save_model(saver, sess, args.ai_input_dir, epsilon=1.0)
epsilon_generator = LinearDecayGreedyEpsilonPolicy(
1.0, args.epsilon, TOTAL_WORKER_JOBS / 5.0)
fits_so_far = 0
mprint('Begin to train (now safe to run gcloud)')
mprint('Initial mean_max_q: ' + str(calculate_mean_max_Q(sess, online_model, fix_samples)))
while len(play_dirs) < TOTAL_WORKER_JOBS:
output_dirs = os.listdir(args.ai_output_dir)
output_dirs = [os.path.join(args.ai_output_dir, x) for x in output_dirs]
output_dirs = set(x for x in output_dirs if os.path.isdir(x))
new_dirs = sorted(output_dirs - evaluation_dirs - play_dirs)
if len(new_dirs) == 0:
time.sleep(0.1)
continue
new_dir = new_dirs[-1] # Most recent gameplay.
evaluation_path = os.path.join(new_dir, WORKER_OUTPUT_EVALUATE_FILENAME)
if os.path.isfile(evaluation_path):
evaluation_dirs.add(new_dir)
with open(evaluation_path, 'rb') as evaluation_file:
rewards, game_lengths, mean_max_Q = pickle.load(evaluation_file)
evaluation = [np.mean(rewards), np.std(rewards),
np.mean(game_lengths), np.std(game_lengths),
mean_max_Q]
mprint('Evaluation: ' + '\t'.join(str(x) for x in evaluation))
continue
memory_path = os.path.join(new_dir, WORKER_OUTPUT_GAMEPLAY_FILENAME)
try:
if os.path.getsize(memory_path) == 0:
# TODO Figure out why this happens despite temporary directory work.
# Also sometimes the file doesn't exist? Hence the try/except.
mprint('Output not ready somehow: ' + memory_path)
time.sleep(0.1)
continue
with open(memory_path, 'rb') as memory_file:
worker_memories = pickle.load(memory_file)
except Exception as exception:
print('Error reading ' + memory_path + ': ' + str(exception.args))
time.sleep(0.1)
continue
for worker_memory in worker_memories:
replay_memory.append(*worker_memory)
if args.psc:
os.remove(memory_path)
play_dirs.add(new_dir)
if len(play_dirs) <= NUM_BURN_IN_JOBS:
mprint('Skip training because still burn in.')
mprint('len(worker_memories): ' + str(len(worker_memories)))
continue
for _ in range(int(len(worker_memories) * FITS_PER_SINGLE_MEMORY)):
agent.fit(sess, fits_so_far)
fits_so_far += 1
# Partial evaluation to give frequent insight into agent progress.
# Last time checked, this took ~0.1 seconds to complete.
mprint('mean_max_q, len(worker_memories): ' +
str(calculate_mean_max_Q(sess, online_model, fix_samples)) +
', ' + str(len(worker_memories)))
# Always decrement epsilon (e.g. not just when saving model).
model_epsilon = epsilon_generator.get_epsilon(decay_epsilon=True)
if len(play_dirs) % SAVE_MODEL_EVERY == 0:
save_model(saver, sess, args.ai_input_dir, model_epsilon)
def fit(self, env, num_iterations, max_episode_length=None):
"""Fit your model to the provided environment.
Its a good idea to print out things like loss, average reward,
Q-values, etc to see if your agent is actually improving.
You should probably also periodically save your network
weights and any other useful info.
This is where you should sample actions from your network,
collect experience samples and add them to your replay memory,
and update your network parameters.
Parameters
----------
env: gym.Env
This is your Atari environment. You should wrap the
environment using the wrap_atari_env function in the
utils.py
num_iterations: int
How many samples/updates to perform.
max_episode_length: int
How long a single episode should last before the agent
resets. Can help exploration.
"""
cnt = np.long(0)
episode_rwd = 0
_screen_raw = self.process_env_reset(env) # Save to history
mse_loss, mae_metric = 0, 0
self.policy = UniformRandomPolicy(env.action_space.n)
evaluation_interval_cnt = 0
while cnt < num_iterations:
cnt += 1
evaluation_interval_cnt += 1
current_state = self.historyPre.get_current_state()
action = self.select_action(current_state, self.q) # Get action
_screen_next_raw, reward, isterminal, _ = env.step(
action) # take action, observe new
episode_rwd += reward
_screen_raw = self.process_one_screen(
_screen_raw, action, reward, _screen_next_raw, isterminal,
True) # Save to history, Memory
# print "\t state: %d, Step: %d, reward: %d, terminal: %d, Observe: %d" \
# % (np.matrix(_screen).sum(), action, reward, isterminal, np.matrix(_screen_next).sum())
# env.render()
if isterminal: # reset
if evaluation_interval_cnt >= self.config.evaluation_interval:
Aver_reward = self.evaluate(env,
self.config.eval_batch_num)
# print ("----------Evaluate, Average reward", Aver_reward)
evaluation_interval_cnt = 0
with open(self.config.rewardlog, "a") as log:
log.write(",".join([
str(int(cnt / self.config.evaluation_interval)),
str(Aver_reward)
]) + "\n")
_screen_raw = self.process_env_reset(env)
# print ("Episode End, iter: ", cnt, "last batch loss: ", mse_loss, 'last mae Metric: ', mae_metric, "Episode reward: ", episode_rwd)
episode_rwd = 0
if cnt >= self.num_burn_in and cnt % self.train_freq == 0: # update
samples = self.AtariPre.process_batch(
self.memory.sample(self.batch_size))
x = np.zeros(
(self.batch_size, self.config.history_length,
self.config.screen_height, self.config.screen_width),
dtype=np.float32)
y = np.zeros((self.batch_size, int(action_size(env))),
dtype=np.float32)
for _index in range(len(samples)):
sample = samples[_index]
x[_index] = np.copy(sample.state)
if sample.is_terminal:
y[_index] = self.calc_q_values(sample.state, self.q)
y[_index][sample.action] = sample.reward
else:
y[_index] = self.calc_q_values(sample.state, self.q)
q_next = max(
self.calc_q_values(
sample.next_state,
self.q_target)) # Use max to update
y[_index][sample.
action] = sample.reward + self.gamma * q_next
mse_loss, mae_metric = self.q.train_on_batch(x, y)
with open(self.config.losslog, "a") as log:
log.write(",".join(
[str(cnt /
4), str(mse_loss),
str(mae_metric)]) + "\n")
# print(cnt, mse_loss, mae_metric)
if cnt % self.config.target_q_update_step == 0: # Set q == q^
self.q_target.set_weights(self.q.get_weights())
if cnt == self.config.memory_size: # change Policy
self.policy = LinearDecayGreedyEpsilonPolicy(
1, 0.05, self.config.decayNum)
if cnt % (num_iterations / 3) == 0: # Save model
TimeStamp = datetime.datetime.strftime(datetime.datetime.now(),
"%y-%m-%d_%H-%M")
self.q.save_weights(
str(self.config.modelname) + '_' + TimeStamp +
'_weights.h5')
return mse_loss, mae_metric, self.q, self.q_target
def main():
# load json and create model
json_file = open(
'/home/shivang/Desktop/HW2TomShivang/deeprl_hw2_src_DQNv2/model.json',
'r')
loaded_model_json = json_file.read()
json_file.close()
model = model_from_json(loaded_model_json)
# load weights into new model
model.load_weights("model.h5")
print("Loaded model from disk")
parser = argparse.ArgumentParser(description='Run DQN on Atari Breakout')
parser.add_argument('--env', default='Breakout-v0', help='Atari env name')
parser.add_argument('-o',
'--output',
default='atari-v0',
help='Directory to save data to')
parser.add_argument('--seed', default=0, type=int, help='Random seed')
args = parser.parse_args()
# args.input_shape = tuple(args.input_shape)
# args.output = get_output_folder(args.output, args.env)
# set up environment model
env1 = gym.make(str(args.env))
NUM_ACTIONS = env1.action_space.n # env.get_action_space().num_actions()
# make dqn agent
FRAMES_PER_STATE = 4
MAX_EPISODE_LEN = 1000
preprocessor = HistoryPreprocessor(FRAMES_PER_STATE - 1)
policy = LinearDecayGreedyEpsilonPolicy(1, .05, int(1e6))
preprocessor = HistoryPreprocessor(FRAMES_PER_STATE - 1)
# evaluate loaded model on test data
#compile agent
adam = Adam(lr=0.0001)
loss = mean_huber_loss
model.compile(loss=loss, optimizer=adam)
max_episode_length = MAX_EPISODE_LEN
num_episodes = 20
"""Test your agent with a provided environment.
You shouldn't update your network parameters here. Also if you
have any layers that vary in behavior between train/test time
(such as dropout or batch norm), you should set them to test.
Basically run your policy on the environment and collect stats
like cumulative reward, average episode length, etc.
You can also call the render function here if you want to
visually inspect your policy.
"""
cumulative_reward = 0
actions = np.zeros(env1.action_space.n)
no_op_max = 30
for episodes in range(num_episodes):
if episodes < 4:
env = wrappers.Monitor(
env1,
'/home/shivang/Desktop/HW2TomShivang/Video_evaluation/' +
str(episodes) + '/',
force=True)
else:
env = env1
# get initial state
preprocessor.reset()
preprocessor.process_state_for_network(env.reset())
state = preprocessor.frames
steps = 0
q_vals_eval = np.zeros(no_op_max)
for i in range(no_op_max):
q_vals = model.predict(state)
(next_state, reward, is_terminal, info) = env.step(0)
preprocessor.process_state_for_network(next_state)
next_state = preprocessor.frames
actions[0] += 1
steps = steps + 1
q_vals_eval[i] = q_vals_eval[i] + max(q_vals[0])
if is_terminal:
state = env.reset()
else:
state = next_state
while steps < max_episode_length:
q_vals = model.predict(state)
action = np.argmax(q_vals[0])
actions[action] += 1
(next_state, reward, is_terminal, info) = env.step(action)
# reward = self.preprocessor.process_reward(reward)
cumulative_reward = cumulative_reward + reward
preprocessor.process_state_for_network(next_state)
next_state = preprocessor.frames
state = next_state
steps = steps + 1
if is_terminal:
break
print(actions)
avg_reward = cumulative_reward / num_episodes
avg_qval = np.mean(q_vals_eval) / num_episodes
print(avg_reward)
print(avg_qval)
def main(args):
# gpu id
# gpu_id = args.gpu
# os.environ['CUDA_VISIBLE_DEVICES'] = '%d'%gpu_id
# make env
env = gym.make(args.env)
if args.mode == 'test' and args.submit:
monitor_log = os.path.join(args.output, 'monitor.log')
env = wrappers.Monitor(env, monitor_log, force=True)
# build model
# actions 0-5: 0 do nothing, 1 fire, 2 right, 3 left, 4 right+fire, 5 left+fire
num_actions = env.action_space.n
mem_size = 1000000
window = 4
input_shape = (84, 84)
if args.type in ['DQN', 'double-DQN']:
model = create_model(window, input_shape, num_actions, args.init)
target = create_model(window, input_shape, num_actions, args.init)
elif args.type in ['linear', 'linear-simple', 'double-Q']:
model = create_model_linear(window, input_shape, num_actions,
args.init)
target = create_model_linear(window, input_shape, num_actions,
args.init)
elif args.type == 'duel':
model = create_model_duel(window, input_shape, num_actions, args.init)
target = create_model_duel(window, input_shape, num_actions, args.init)
# memory = ReplayMemory(1000000, 100) # window length is arbitrary
# target_update_freq = 10000
# num_burn_in = 50000
target_update_freq = 10000
num_burn_in = 50000
train_freq = 4
batch_size = 32
gamma = 0.99
epsilon = 0.05
updates_per_epoch = 50000
num_iterations = 50000000
eval_episodes = 100
max_episode_length = 10000
# simple: no experience replay and no target fixing
# if args.type == 'linear-simple':
# mem_size = 5
# target_update_freq = 1
# num_burn_in = 0
# batch_size = 1
if args.type == 'linear-simple':
num_burn_in = 0
memory = ReplayMemoryEfficient(mem_size, window, input_shape)
# with tf.device('/gpu:%d'%gpu_id):
config = tf.ConfigProto(intra_op_parallelism_threads=8)
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
# preprocessor
preprocessor = PreprocessorSequence()
# policy
policy = LinearDecayGreedyEpsilonPolicy(1, 0.1, 1000000)
policy_eval = GreedyEpsilonPolicy(epsilon)
# build agent
dqn_agent = DQNAgent(sess, env, args.type, model, target, preprocessor,
memory, policy, policy_eval, gamma,
target_update_freq, num_burn_in, train_freq,
batch_size, num_actions, updates_per_epoch,
args.output)
if args.mode == 'train': # compile net and train with fit
# rmsprop = RMSprop(lr=0.00025, rho=0.95, epsilon=0.01)
# dqn_agent.compile_networks(rmsprop, mean_huber_loss)
# adam = Adam(lr=0.00025, beta_1=0.95, beta_2=0.95, epsilon=0.1)
adam = Adam(lr=0.0001)
dqn_agent.compile_networks(adam, mean_huber_loss)
if args.type == 'linear-simple':
dqn_agent.fit_simple(num_iterations, max_episode_length)
else:
dqn_agent.fit(num_iterations, max_episode_length)
elif args.mode == 'test': # load net and evaluate
model_path = os.path.join(args.output, 'model_epoch%03d' % args.epoch)
dqn_agent.load_networks(model_path)
if args.submit:
eval_episodes = 1
dqn_agent.play(eval_episodes, max_episode_length)
# if args.submit:
# gym.upload(monitor_log, api_key='sk_wa5MgeDTnOQ209qBCP7jQ')
# else:
# log_file = open(os.path.join(args.output, 'evaluation.txt'), 'a+')
# log_file.write('%d %f %f %f %f\n' % (args.epoch,
# np.mean(lengths),
# np.std(lengths),
# np.mean(rewards),
# np.std(rewards)))
# log_file.close()
env.close()
def main():
parser = argparse.ArgumentParser(description='Run DQN on Atari Breakout')
parser.add_argument('--env', default='Breakout-v0', help='Atari env name')
parser.add_argument('-o',
'--output',
default='atari-v0',
help='Directory to save data to')
parser.add_argument('--seed', default=0, type=int, help='Random seed')
parser.add_argument('--mode', choices=['train', 'test'], default='test')
parser.add_argument('--network',
choices=['deep', 'linear'],
default='deep')
parser.add_argument('--method',
choices=['dqn', 'double', 'dueling'],
default='dqn')
parser.add_argument('--monitor', type=bool, default=True)
parser.add_argument('--iter', type=int, default=2400000)
parser.add_argument('--test_policy',
choices=['Greedy', 'GreedyEpsilon'],
default='GreedyEpsilon')
args = parser.parse_args()
args.seed = np.random.randint(0, 1000000, 1)[0]
args.weights = 'models/dqn_{}_weights_{}_{}_{}.h5f'.format(
args.env, args.method, args.network, args.iter)
args.monitor_path = 'tmp/dqn_{}_weights_{}_{}_{}_{}'.format(
args.env, args.method, args.network, args.iter, args.test_policy)
if args.mode == 'train':
args.monitor = False
env = gym.make(args.env)
if args.monitor:
env = wrappers.Monitor(env, args.monitor_path)
np.random.seed(args.seed)
env.seed(args.seed)
args.gamma = 0.99
args.learning_rate = 0.0001
args.epsilon = 0.05
args.num_iterations = 5000000
args.batch_size = 32
args.window_length = 4
args.num_burn_in = 50000
args.target_update_freq = 10000
args.log_interval = 10000
args.model_checkpoint_interval = 10000
args.train_freq = 4
args.num_actions = env.action_space.n
args.input_shape = (84, 84)
args.memory_max_size = 1000000
args.output = get_output_folder(args.output, args.env)
args.suffix = args.method + '_' + args.network
if (args.method == 'dqn'):
args.enable_double_dqn = False
args.enable_dueling_network = False
elif (args.method == 'double'):
args.enable_double_dqn = True
args.enable_dueling_network = False
elif (args.method == 'dueling'):
args.enable_double_dqn = False
args.enable_dueling_network = True
else:
print('Attention! Method Worng!!!')
if args.test_policy == 'Greedy':
test_policy = GreedyPolicy()
elif args.test_policy == 'GreedyEpsilon':
test_policy = GreedyEpsilonPolicy(args.epsilon)
print(args)
K.tensorflow_backend.set_session(get_session())
model = create_model(args.window_length, args.input_shape,
args.num_actions, args.network)
# we create our preprocessor, the Ataripreprocessor will only process current frame the agent is seeing. And the sequence
# preprocessor will construct the state by concatenating 3 previous frames from HistoryPreprocessor and current processed frame
Processor = {}
Processor['Atari'] = AtariPreprocessor(args.input_shape)
Processor['History'] = HistoryPreprocessor(args.window_length)
ProcessorSequence = PreprocessorSequence(Processor) # construct 84x84x4
# we create our memory for saving all experience collected during training with window length 4
memory = ReplayMemory(max_size=args.memory_max_size,
input_shape=args.input_shape,
window_length=args.window_length)
# we use linear decay greedy epsilon policy and tune the epsilon from 1 to 0.1 during the first 100w iterations and then keep using
# epsilon with 0.1 to further train the network
policy = LinearDecayGreedyEpsilonPolicy(GreedyEpsilonPolicy(args.epsilon),
attr_name='eps',
start_value=1,
end_value=0.1,
num_steps=1000000)
# we construct our agent and use 0.99 as our discounted factor, 32 as our batch_size. We update our model for each 4 iterations. But during first
# 50000 iterations, we only collect data to the memory and don't update our model.
dqn = DQNAgent(q_network=model,
policy=policy,
memory=memory,
num_actions=args.num_actions,
test_policy=test_policy,
preprocessor=ProcessorSequence,
gamma=args.gamma,
target_update_freq=args.target_update_freq,
num_burn_in=args.num_burn_in,
train_freq=args.train_freq,
batch_size=args.batch_size,
enable_double_dqn=args.enable_double_dqn,
enable_dueling_network=args.enable_dueling_network)
adam = Adam(lr=args.learning_rate)
dqn.compile(optimizer=adam)
if args.mode == 'train':
weights_filename = 'dqn_{}_weights_{}.h5f'.format(
args.env, args.suffix)
checkpoint_weights_filename = 'dqn_' + args.env + '_weights_' + args.suffix + '_{step}.h5f'
log_filename = 'dqn_{}_log_{}.json'.format(args.env, args.suffix)
log_dir = '../tensorboard_{}_log_{}'.format(args.env, args.suffix)
callbacks = [
ModelIntervalCheckpoint(checkpoint_weights_filename,
interval=args.model_checkpoint_interval)
]
callbacks += [FileLogger(log_filename, interval=100)]
callbacks += [
TensorboardStepVisualization(log_dir=log_dir,
histogram_freq=1,
write_graph=True,
write_images=True)
]
# start training
# we don't apply action repetition explicitly since the game will randomly skip frame itself
dqn.fit(env,
callbacks=callbacks,
verbose=1,
num_iterations=args.num_iterations,
action_repetition=1,
log_interval=args.log_interval,
visualize=True)
dqn.save_weights(weights_filename, overwrite=True)
dqn.evaluate(env,
num_episodes=10,
visualize=True,
num_burn_in=5,
action_repetition=1)
elif args.mode == 'test':
weights_filename = 'dqn_{}_weights_{}.h5f'.format(
args.env, args.suffix)
if args.weights:
weights_filename = args.weights
dqn.load_weights(weights_filename)
dqn.evaluate(env,
num_episodes=250,
visualize=True,
num_burn_in=5,
action_repetition=1)
# we upload our result to openai gym
if args.monitor:
env.close()
gym.upload(args.monitor_path, api_key='sk_J62obX9PQg2ExrM6H9rvzQ')
C = 10000
env = gym.make("Breakout-v0")
#number of actions, used to construct an policy selector
num_actions = env.action_space.n
#create helpers
#observation processor
atari_processor = AtariProcessor(IMAGE_SIZE)
history_store = HistoryStore(HISTORY_LENGTH, IMAGE_SIZE)
#policy selector, for testing, use uniform random policy selector, just pass number of actions to the constructor
random_selector = UniformRandomPolicy(num_actions)
greedy_selector = GreedyPolicy()
greedy_epsilon_selector = GreedyEpsilonPolicy(epsilon)
greedy_epsilon_linear_decay_selector = LinearDecayGreedyEpsilonPolicy(
1, 0.05, int(round(MAX_INTERACTION / 5, 0)))
# Initialize neural network
# Online network which changes during training but not to calculate Q*.
model_online = NN_cnn((IMAGE_SIZE[0], IMAGE_SIZE[1], HISTORY_LENGTH),
num_actions)
# Fixed network which is not changed during training but to calculate Q*.
model_fixed = NN_cnn((IMAGE_SIZE[0], IMAGE_SIZE[1], HISTORY_LENGTH),
num_actions)
model_fixed.model.set_weights(model_online.model.get_weights())
#model_fixed.model = Model.from_config(model_online.model.get_config())
# Initialize memory.
mem = NNMemStore(MEM_SIZE, (IMAGE_SIZE[0], IMAGE_SIZE[1], HISTORY_LENGTH))
mem.fill_half(env, random_selector, atari_processor, history_store, "matrix")
def main():
#env = gym.make("Enduro-v0")
#env = gym.make("SpaceInvaders-v0")
#env = gym.make("Breakout-v0")
model_name = "result-q6-qqdn"
if (len(sys.argv) >= 2):
model_name = sys.argv[1]
if (len(sys.argv) >= 3):
env = gym.make(sys.argv[2])
else:
#env = gym.make("Enduro-v0")
env = gym.make("SpaceInvaders-v0")
#env = gym.make("Breakout-v0")
#no skip frames
env.frameskip = 1
input_shape = (84, 84)
batch_size = 32
num_actions = env.action_space.n
memory_size = 1000000
memory_burn_in_num = 50000
start_epsilon = 1
end_epsilon = 0.01
decay_steps = 1000000
target_update_freq = 10000
train_freq = 4 #How often you train the network
history_size = 4
history_prep = HistoryPreprocessor(history_size)
atari_prep = AtariPreprocessor(input_shape, 0, 999)
numpy_prep = NumpyPreprocessor()
preprocessors = PreprocessorSequence(
[atari_prep, history_prep, numpy_prep]) #from left to right
policy = LinearDecayGreedyEpsilonPolicy(start_epsilon, end_epsilon,
decay_steps)
model = create_model(history_size, input_shape, num_actions, model_name)
model.summary()
#plot_model(model,to_file="dueling.png")
optimizer = Adam(lr=0.001,
beta_1=0.9,
beta_2=0.999,
epsilon=1e-08,
decay=0.0)
loss_func = huber_loss
#linear_model.compile(optimizer, loss_func)
random_policy = UniformRandomPolicy(num_actions)
#memory = ActionReplayMemory(1000000,4)
memory = ActionReplayMemory(memory_size, 4)
memory_burn_in(env, memory, preprocessors, memory_burn_in_num,
random_policy)
#print(reward_arr)
#print(curr_state_arr)
agent = DDQNAgent(model, preprocessors, memory, policy, 0.99,
target_update_freq, None, train_freq, batch_size)
agent.compile(optimizer, loss_func)
agent.save_models()
agent.fit(env, 1000000, 100000)
def main(): # noqa: D103
parser = argparse.ArgumentParser(description='Run DQN on Atari Breakout')
parser.add_argument('--env',
default='SpaceInvadersDeterministic-v3',
help='Atari env name')
parser.add_argument('-o',
'--output',
default='atari-v0',
help='Directory to save data to')
parser.add_argument('--seed', default=0, type=int, help='Random seed')
parser.add_argument('--model',
default='dqn',
help='Q Network type to use.')
parser.add_argument('--double', action='store_true')
model_map = {
'linear': LinearQN,
'mlp': MLP,
'dqn': DQN,
'dueling': DuelingDQN
}
args = parser.parse_args()
args.model = args.model.lower()
if args.model not in model_map:
print("Invalid model type. Valid types are", model_map.keys())
sys.exit(1)
args.output = get_output_folder(args.output, args.env)
# here is where you should start up a session,
# create your DQN agent, create your model, etc.
# then you can run your fit method.
env = gym.make(args.env)
monitored_env = gym.wrappers.Monitor(
gym.make(args.env),
args.output,
video_callable=lambda i: i % EVAL_NUM_EPISODES == 0)
atari = not args.env.startswith("CartPole")
if atari:
input_shape = (IMAGE_SIZE, IMAGE_SIZE)
preprocessor = lambda: PreprocessorSequence(
AtariPreprocessor(new_size=input_shape),
HistoryPreprocessor(history_length=WINDOW_SIZE, max_over=True))
else:
input_shape = (4, )
preprocessor = lambda: HistoryPreprocessor(history_length=WINDOW_SIZE)
memory = ExperienceReplay(max_size=REPLAY_BUFFER_SIZE,
window_length=WINDOW_SIZE)
NUM_ACTIONS = env.action_space.n
#policy = UniformRandomPolicy(num_actions=NUM_ACTIONS)
#policy = GreedyEpsilonPolicy(NUM_ACTIONS, EPSILON)
policy = LinearDecayGreedyEpsilonPolicy(NUM_ACTIONS, 1.0, EPSILON,
NUM_ITERATIONS_LINEAR_DECAY)
model = model_map[args.model](exp_name=args.output)
agent = DQNAgent(q_network=model,
preprocessor=preprocessor,
memory=memory,
policy=policy,
gamma=GAMMA,
target_update_freq=TARGET_UPDATE_FREQ,
replay_buffer_size=REPLAY_BUFFER_SIZE,
train_freq=TRAIN_FREQ,
batch_size=BATCH_SIZE,
output_dir=args.output,
double_dqn=args.double)
agent.compile(window=WINDOW_SIZE,
input_shape=input_shape,
num_actions=NUM_ACTIONS,
model_name='q_network')
signal.signal(signal.SIGINT, agent.signal_handler)
signal.signal(signal.SIGTERM, agent.signal_handler)
signal.signal(signal.SIGHUP, agent.signal_handler)
agent.fit(env, monitored_env, num_iterations=NUM_ITERATIONS)
def testAgent():
parser = argparse.ArgumentParser(description='Run DQN on Atari Breakout')
parser.add_argument('--env', default='SpaceInvaders-v0', help='Atari env name')
parser.add_argument(
'-o', '--output', default='atari-v0', help='Directory to save data to')
parser.add_argument('--seed', default=0, type=int, help='Random seed')
parser.add_argument('--input_shape', default=(84,84), type=int, help='Input shape')
parser.add_argument('--phase', default='train', type=str, help='Train/Test/Video')
parser.add_argument('-r', '--render', action='store_true', default=False, help='Render')
parser.add_argument('--model', default='deep_Q_network', type=str, help='Type of model')
parser.add_argument('-c', action='store_false', default=True, help='Cancel')
parser.add_argument('-d', '--dir', default='', type=str, help='Directory')
parser.add_argument('-n', '--number', default='', type=str, help='Model number')
args = parser.parse_args()
assert(args.phase in ['train', 'test', 'video'])
assert(args.dir if args.phase == 'test' or args.phase == 'video' else True)
args.input_shape = tuple(args.input_shape)
# create the environment
env = gym.make(args.env)
# Number of training iterations
num_iterations = 5000000
# Learning rate
alpha = 0.0001
# Epsilion for GreedyEpsilonPolicy
epsilon = 0.05
# Parameters for LinearDecayGreedyEpsilonPolicy
start_value = 0.3
end_value = 0.05
num_steps = 10000
# Number of frames in the sequence
window = 4
# Use experience replay
experience_replay = args.c
# Use target fixing
target_fixing = args.c
# Evaluate number of episode (given the model number)
num_episode = 1
# DQNAgent parameters
num_actions = env.action_space.n
q_network = create_model(window,
args.input_shape,
num_actions,
model_name=args.model)
preprocessor = AtariPreprocessor(args.input_shape)
policy = LinearDecayGreedyEpsilonPolicy(num_actions, start_value, end_value, num_steps)
memory_size = 1000000
gamma = 0.99
target_update_freq = 100
num_burn_in = 50
train_freq = 4
batch_size = 32
video_capture_points = (num_iterations * np.array([0/3., 1/3., 2/3., 3/3.])).astype('int')
save_network_freq = 100
eval_train_freq = 50000
eval_train_num_ep = 1
if experience_replay:
memory = BasicMemory(memory_size, window)
else:
memory = NaiveMemory(batch_size, window)
dqnAgent = DQNAgent(args.model,
q_network,
preprocessor,
memory,
policy,
gamma,
target_update_freq,
num_burn_in,
train_freq,
batch_size,
num_actions,
window,
save_network_freq,
video_capture_points,
eval_train_freq,
eval_train_num_ep,
args.phase,
target_fixing=target_fixing,
render=args.render)
q_values = np.array([[1.1, 1.2, 1.3, 1.4, 1.5, 1.7], \
[1.3, 1.4, 1.5, 1.6, 1.1, 1.2], \
[1.2, 1.3, 1.4, 1.5, 2.2, 1.1], \
[1.5, 3.8, 1.1, 1.2, 1.3, 1.4], \
[0, 0, 0, 0.7, 0, 0]])
is_terminal = np.array([0, 0, 1, 0, 1])
reward = np.array([0.4, 0.5, 0.6, 0.7, 0.8])
target = dqnAgent.calc_target_values(q_values, is_terminal, reward)
assert(np.array_equal(target, np.array([2.083, 2.084, 0.6, 4.462, 0.8])))
bm = BasicMemory(10, 3)
bm.append(np.array([[0,0],[0,0]]), 0, 1, False)
bm.append(np.array([[1,1],[1,1]]), 1, 1, False)
bm.append(np.array([[2,2],[2,2]]), 2, 1, False)
bm.append(np.array([[3,3],[3,3]]), 3, 1, True)
bm.append(np.array([[4,4],[4,4]]), 0, 1, False)
bm.append(np.array([[5,5],[5,5]]), 1, 1, False)
bm.append(np.array([[6,6],[6,6]]), 2, 1, True)
bm.append(np.array([[7,7],[7,7]]), 3, 1, False)
bm.append(np.array([[8,8],[8,8]]), 0, 1, False)
bm.append(np.array([[9,9],[9,9]]), 1, 1, False)
bm.append(np.array([[10,10],[10,10]]), 2, 1, False)
bm.append(np.array([[11,11],[11,11]]), 3, 1, False)
bm.append(np.array([[12,12],[12,12]]), 0, 1, False)
minibatch = bm.sample(5, indexes=[0, 4, 5, 8, 9])
state_batch, \
action_batch, \
reward_batch, \
next_state_batch, \
is_terminal_batch = dqnAgent.process_batch(minibatch)
assert(np.array_equal(state_batch, np.array([[[[8.,9.,10.], \
[8.,9.,10.]], \
[[8.,9.,10.], \
[8.,9.,10.]]], \
[[[0.,0.,4.], \
[0.,0.,4.]], \
[[0.,0.,4.], \
[0.,0.,4.]]], \
[[[0.,4.,5.], \
[0.,4.,5.]], \
[[0.,4.,5.], \
[0.,4.,5.]]], \
[[[0.,7.,8.], \
[0.,7.,8.]], \
[[0.,7.,8.], \
[0.,7.,8.]]], \
[[[7.,8.,9.], \
[7.,8.,9.]], \
[[7.,8.,9.], \
[7.,8.,9.]]]])))
assert(np.array_equal(action_batch, np.array([2, 0, 1, 0, 1])))
assert(np.array_equal(reward_batch, np.array([1, 1, 1, 1, 1])))
assert(np.array_equal(next_state_batch, np.array([[[[9.,10.,11.], \
[9.,10.,11.]], \
[[9.,10.,11.], \
[9.,10.,11.]]], \
[[[0.,4.,5.], \
[0.,4.,5.]], \
[[0.,4.,5.], \
[0.,4.,5.]]], \
[[[4.,5.,6.], \
[4.,5.,6.]], \
[[4.,5.,6.], \
[4.,5.,6.]]], \
[[[7.,8.,9.], \
[7.,8.,9.]], \
[[7.,8.,9.], \
[7.,8.,9.]]], \
[[[8.,9.,10.], \
[8.,9.,10.]], \
[[8.,9.,10.], \
[8.,9.,10.]]]])))
assert(np.array_equal(is_terminal_batch, np.array([False, False, False, False, False])))