def main():
if (len(sys.argv) != 6):
print("usage:{} <env> <model_json> <weights> <render> <random>".format(
sys.argv[0]))
return sys.exit()
env = gym.make(sys.argv[1])
env.frameskip = 1
with open(sys.argv[2]) as json_file:
model = model_from_json(json.load(json_file), {"Eq9": Eq9})
model.load_weights(sys.argv[3])
epsilon = 0.01
input_shape = (84, 84)
history_size = 4
eval_size = 100
render = (sys.argv[4] == "y")
history_prep = HistoryPreprocessor(history_size)
atari_prep = AtariPreprocessor(input_shape, 0, 999)
numpy_prep = NumpyPreprocessor()
preprocessors = PreprocessorSequence(
[atari_prep, history_prep, numpy_prep]) #from left to right
if (sys.argv[5] == "y"):
print("using random policy")
policy = UniformRandomPolicy(env.action_space.n)
else:
print("using greedy policy")
policy = GreedyEpsilonPolicy(epsilon)
agent = DQNAgent(model, preprocessors, None, policy, 0.99, None, None,
None, None)
agent.add_keras_custom_layers({"Eq9": Eq9})
reward_arr, length_arr = agent.evaluate_detailed(env,
eval_size,
render=render,
verbose=True)
print("\rPlayed {} games, reward:M={}, SD={} length:M={}, SD={}".format(
eval_size, np.mean(reward_arr), np.std(reward_arr),
np.mean(length_arr), np.std(reward_arr)))
print("max:{} min:{}".format(np.max(reward_arr), np.min(reward_arr)))
plt.hist(reward_arr)
plt.show()
def main():
#env = gym.make("Enduro-v0")
#env = gym.make("SpaceInvaders-v0")
#env = gym.make("Breakout-v0")
model_name = "q2"
if (len(sys.argv) >= 2):
model_name = sys.argv[1]
if (len(sys.argv) >= 3):
env = gym.make(sys.argv[2])
else:
#env = gym.make("Enduro-v0")
env = gym.make("SpaceInvaders-v0")
#env = gym.make("Breakout-v0")
#no skip frames
env.frameskip = 1
input_shape = (84, 84)
batch_size = 1
num_actions = env.action_space.n
memory_size = 2 #2 because it need to save the current state and the future state, no matter what it gets, it will always just pick the earlier one
memory_burn_in_num = 1
start_epsilon = 1
end_epsilon = 0.01
decay_steps = 1000000
target_update_freq = 1 #no targeting
train_freq = 4 #How often you train the network
history_size = 4
history_prep = HistoryPreprocessor(history_size)
atari_prep = AtariPreprocessor(input_shape, 0, 999)
numpy_prep = NumpyPreprocessor()
preprocessors = PreprocessorSequence(
[atari_prep, history_prep, numpy_prep]) #from left to right
policy = LinearDecayGreedyEpsilonPolicy(start_epsilon, end_epsilon,
decay_steps)
linear_model = create_model(history_size, input_shape, num_actions,
model_name)
optimizer = Adam(lr=0.001,
beta_1=0.9,
beta_2=0.999,
epsilon=1e-08,
decay=0.0)
loss_func = huber_loss
#linear_model.compile(optimizer, loss_func)
linear_model.summary()
random_policy = UniformRandomPolicy(num_actions)
#memory = ActionReplayMemory(1000000,4)
memory = ActionReplayMemory(memory_size, history_size)
#memory_burn_in(env,memory,preprocessors,memory_burn_in_num,random_policy)
#print(reward_arr)
#print(curr_state_arr)
agent = DQNAgent(linear_model, preprocessors, memory, policy, 0.99,
target_update_freq, None, train_freq, batch_size)
agent.compile(optimizer, loss_func)
agent.save_models()
agent.fit(env, 1000000, 100000)
def main(): # noqa: D103
#(SpaceInvaders-v0
# Enduro-v0
parser = argparse.ArgumentParser(description='Run DQN on Atari Breakout')
parser.add_argument('--env',
default='SpaceInvaders-v0',
help='Atari env name')
#parser.add_argument('--env', default='SpaceInvaders-v0', help='Atari env name')
#parser.add_argument('--env', default='PendulumSai-v0', help='Atari env name')
parser.add_argument('-o',
'--output',
default='atari-v0',
help='Directory to save data to')
parser.add_argument('--seed', default=0, type=int, help='Random seed')
args = parser.parse_args()
#args.input_shape = tuple(args.input_shape)
#args.output = get_output_folder(args.output, args.env)
# here is where you should start up a session,
# create your DQN agent, create your model, etc.
# then you can run your fit method.
model_name = 'linear'
env = gym.make(args.env)
num_iter = 2000000
max_epi_iter = 1000
epsilon = 0.4
window = 4
gamma = 0.99
target_update_freq = 5000
train_freq = 1
batch_size = 32
num_burn_in = 5000
num_actions = 3 #env.action_space.n
state_size = (84, 84, 1)
new_size = state_size
max_size = 1000000
lr = 0.00020
beta_1 = 0.9
beta_2 = 0.999
epsilon2 = 1e-08
decay = 0.0
u_policy = UniformRandomPolicy(num_actions)
ge_policy = GreedyEpsilonPolicy(epsilon)
g_policy = GreedyPolicy()
policy = {
'u_policy': u_policy,
'ge_policy': ge_policy,
'g_policy': g_policy
}
#preprocessor = PreprocessorSequence([AtariPreprocessor(new_size), HistoryPreprocessor(window)])
preprocessor = AtariPreprocessor(new_size)
memory = SequentialMemory(max_size=max_size, window_length=window)
model = create_model(window, state_size, num_actions)
print(model.summary())
dqnA = DQNAgent(q_network=model,
preprocessor=preprocessor,
memory=memory,
policy=policy,
gamma=gamma,
target_update_freq=target_update_freq,
num_burn_in=num_burn_in,
train_freq=train_freq,
batch_size=batch_size,
model_name=model_name)
#testing
#selected_action = dqnA.select_action( np.random.rand(1,210,160,12), train=1, warmup_phase=0)
h_loss = huber_loss
optimizer = Adam(lr=lr,
beta_1=beta_1,
beta_2=beta_2,
epsilon=epsilon2,
decay=decay)
dqnA.compile(optimizer, h_loss)
#callback1 = ProgbarLogger(count_mode='samples')
dqnA.fit(env, num_iterations=num_iter, max_episode_length=max_epi_iter)
def fit(self, env, num_iterations, max_episode_length=None):
"""Fit your model to the provided environment.
Its a good idea to print out things like loss, average reward,
Q-values, etc to see if your agent is actually improving.
You should probably also periodically save your network
weights and any other useful info.
This is where you should sample actions from your network,
collect experience samples and add them to your replay memory,
and update your network parameters.
Parameters
----------
env: gym.Env
This is your Atari environment. You should wrap the
environment using the wrap_atari_env function in the
utils.py
num_iterations: int
How many samples/updates to perform.
max_episode_length: int
How long a single episode should last before the agent
resets. Can help exploration.
"""
cnt = np.long(0)
episode_rwd = 0
_screen_raw = self.process_env_reset(env) # Save to history
mse_loss, mae_metric = 0, 0
self.policy = UniformRandomPolicy(env.action_space.n)
evaluation_interval_cnt = 0
while cnt < num_iterations:
cnt += 1
evaluation_interval_cnt += 1
current_state = self.historyPre.get_current_state()
action = self.select_action(current_state, self.q) # Get action
_screen_next_raw, reward, isterminal, _ = env.step(
action) # take action, observe new
episode_rwd += reward
_screen_raw = self.process_one_screen(
_screen_raw, action, reward, _screen_next_raw, isterminal,
True) # Save to history, Memory
# print "\t state: %d, Step: %d, reward: %d, terminal: %d, Observe: %d" \
# % (np.matrix(_screen).sum(), action, reward, isterminal, np.matrix(_screen_next).sum())
# env.render()
if isterminal: # reset
if evaluation_interval_cnt >= self.config.evaluation_interval:
Aver_reward = self.evaluate(env,
self.config.eval_batch_num)
# print ("----------Evaluate, Average reward", Aver_reward)
evaluation_interval_cnt = 0
with open(self.config.rewardlog, "a") as log:
log.write(",".join([
str(int(cnt / self.config.evaluation_interval)),
str(Aver_reward)
]) + "\n")
_screen_raw = self.process_env_reset(env)
# print ("Episode End, iter: ", cnt, "last batch loss: ", mse_loss, 'last mae Metric: ', mae_metric, "Episode reward: ", episode_rwd)
episode_rwd = 0
if cnt >= self.num_burn_in and cnt % self.train_freq == 0: # update
samples = self.AtariPre.process_batch(
self.memory.sample(self.batch_size))
x = np.zeros(
(self.batch_size, self.config.history_length,
self.config.screen_height, self.config.screen_width),
dtype=np.float32)
y = np.zeros((self.batch_size, int(action_size(env))),
dtype=np.float32)
for _index in range(len(samples)):
sample = samples[_index]
x[_index] = np.copy(sample.state)
if sample.is_terminal:
y[_index] = self.calc_q_values(sample.state, self.q)
y[_index][sample.action] = sample.reward
else:
y[_index] = self.calc_q_values(sample.state, self.q)
q_next = max(
self.calc_q_values(
sample.next_state,
self.q_target)) # Use max to update
y[_index][sample.
action] = sample.reward + self.gamma * q_next
mse_loss, mae_metric = self.q.train_on_batch(x, y)
with open(self.config.losslog, "a") as log:
log.write(",".join(
[str(cnt /
4), str(mse_loss),
str(mae_metric)]) + "\n")
# print(cnt, mse_loss, mae_metric)
if cnt % self.config.target_q_update_step == 0: # Set q == q^
self.q_target.set_weights(self.q.get_weights())
if cnt == self.config.memory_size: # change Policy
self.policy = LinearDecayGreedyEpsilonPolicy(
1, 0.05, self.config.decayNum)
if cnt % (num_iterations / 3) == 0: # Save model
TimeStamp = datetime.datetime.strftime(datetime.datetime.now(),
"%y-%m-%d_%H-%M")
self.q.save_weights(
str(self.config.modelname) + '_' + TimeStamp +
'_weights.h5')
return mse_loss, mae_metric, self.q, self.q_target
def main(): # noqa: D103
parser = argparse.ArgumentParser(description='Run DQN on Atari Space Invaders')
parser.add_argument('--seed', default=10703, type=int, help='Random seed')
parser.add_argument('--input_shape', default=SIZE_OF_STATE, help='Input shape')
parser.add_argument('--gamma', default=0.99, help='Discount factor')
# TODO experiment with this value.
parser.add_argument('--epsilon', default=0.1, help='Final exploration probability in epsilon-greedy')
parser.add_argument('--learning_rate', default=0.00025, help='Training learning rate.')
parser.add_argument('--batch_size', default=32, type = int, help=
'Batch size of the training part')
parser.add_argument('--question', type=int, default=7,
help='Which hw question to run.')
parser.add_argument('--evaluate', action='store_true',
help='Only affects worker. Run evaluation instead of training.')
parser.add_argument('--worker_epsilon', type=float,
help='Only affects worker. Override epsilon to use (instead of one in file).')
parser.add_argument('--skip_model_restore', action='store_true',
help='Only affects worker. Use a newly initialized model instead of restoring one.')
parser.add_argument('--generate_fixed_samples', action='store_true',
help=('Special case execution. Generate fixed samples and close. ' +
'This is necessary to run whenever the network or action space changes.'))
parser.add_argument('--ai_input_dir', default='gcloud/inputs/',
help='Input directory with initialization files.')
parser.add_argument('--ai_output_dir', default='gcloud/outputs/',
help='Output directory for gameplay files.')
parser.add_argument('--is_worker', dest='is_manager',
action='store_false',
help='Whether this is a worker (no training).')
parser.add_argument('--is_manager', dest='is_manager',
action='store_true',
help='Whether this is a manager (trains).')
parser.set_defaults(is_manager=True)
parser.add_argument('--psc', action='store_true',
help=('Only affects manager. Whether on PSC, ' +
'and should for example reduce disk usage.'))
# Copied from original phillip code (run.py).
for opt in CPU.full_opts():
opt.update_parser(parser)
parser.add_argument("--dolphin", action="store_true", default=None, help="run dolphin")
for opt in DolphinRunner.full_opts():
opt.update_parser(parser)
args = parser.parse_args()
# run.sh might pass these in via environment variable, so user directory
# might not already be expanded.
args.ai_input_dir = os.path.expanduser(args.ai_input_dir)
args.ai_output_dir = os.path.expanduser(args.ai_output_dir)
if args.is_manager:
random.seed(args.seed)
np.random.seed(args.seed)
tf.set_random_seed(args.seed)
do_evaluation = args.evaluate or random.random() < WORKER_EVALUATION_PROBABILITY
if do_evaluation or args.generate_fixed_samples:
args.cpu = EVAL_CPU_LEVEL
print('OVERRIDING cpu level to: ' + str(EVAL_CPU_LEVEL))
if args.generate_fixed_samples and args.is_manager:
raise Exception('Can not generate fixed samples as manager. Must use ' +
'--is_worker and all other necessary flags (e.g. --iso ISO_PATH)')
env = SmashEnv()
if not args.is_manager:
env.make(args) # Opens Dolphin.
question_settings = get_question_settings(args.question, args.batch_size)
online_model, online_params = create_model(
input_shape=args.input_shape,
num_actions=env.action_space.n, model_name='online_model',
create_network_fn=question_settings['create_network_fn'],
learning_rate=args.learning_rate)
target_model = online_model
update_target_params_ops = []
if (question_settings['target_update_freq'] is not None or
question_settings['is_double_network']):
target_model, target_params = create_model(
input_shape=args.input_shape,
num_actions=env.action_space.n, model_name='target_model',
create_network_fn=question_settings['create_network_fn'],
learning_rate=args.learning_rate)
update_target_params_ops = [t.assign(s) for s, t in zip(online_params, target_params)]
replay_memory = ReplayMemory(
max_size=question_settings['replay_memory_size'],
error_if_full=(not args.is_manager))
saver = tf.train.Saver(max_to_keep=None)
agent = DQNAgent(online_model=online_model,
target_model = target_model,
memory=replay_memory,
gamma=args.gamma,
target_update_freq=question_settings['target_update_freq'],
update_target_params_ops=update_target_params_ops,
batch_size=args.batch_size,
is_double_network=question_settings['is_double_network'],
is_double_dqn=question_settings['is_double_dqn'])
sess = tf.Session()
with sess.as_default():
if args.generate_fixed_samples:
print('Generating ' + str(NUM_FIXED_SAMPLES) + ' fixed samples and saving to ./' + FIXED_SAMPLES_FILENAME)
print('This file is only ever used on the manager.')
agent.compile(sess)
fix_samples = agent.prepare_fixed_samples(
env, sess, UniformRandomPolicy(env.action_space.n),
NUM_FIXED_SAMPLES, MAX_EPISODE_LENGTH)
env.terminate()
with open(FIXED_SAMPLES_FILENAME, 'wb') as f:
pickle.dump(fix_samples, f)
return
if args.is_manager or args.skip_model_restore:
agent.compile(sess)
else:
saver.restore(sess, os.path.join(args.ai_input_dir, WORKER_INPUT_MODEL_FILENAME))
print('_________________')
print('number_actions: ' + str(env.action_space.n))
# Worker code.
if not args.is_manager:
print('ai_input_dir: ' + args.ai_input_dir)
print('ai_output_dir: ' + args.ai_output_dir)
if do_evaluation:
evaluation = agent.evaluate(env, sess, GreedyPolicy(), EVAL_EPISODES, MAX_EPISODE_LENGTH)
print('Evaluation: ' + str(evaluation))
with open(FIXED_SAMPLES_FILENAME, 'rb') as fixed_samples_f:
fix_samples = pickle.load(fixed_samples_f)
mean_max_Q = calculate_mean_max_Q(sess, online_model, fix_samples)
evaluation = evaluation + (mean_max_Q,)
with open(os.path.join(args.ai_output_dir, WORKER_OUTPUT_EVALUATE_FILENAME), 'wb') as f:
pickle.dump(evaluation, f)
env.terminate()
return
worker_epsilon = args.worker_epsilon
if worker_epsilon is None:
with open(os.path.join(args.ai_input_dir, WORKER_INPUT_EPSILON_FILENAME)) as f:
lines = f.readlines()
# TODO handle unexpected lines better than just ignoring?
worker_epsilon = float(lines[0])
print('Worker epsilon: ' + str(worker_epsilon))
train_policy = GreedyEpsilonPolicy(worker_epsilon)
agent.play(env, sess, train_policy, total_seconds=PLAY_TOTAL_SECONDS, max_episode_length=MAX_EPISODE_LENGTH)
replay_memory.save_to_file(os.path.join(args.ai_output_dir, WORKER_OUTPUT_GAMEPLAY_FILENAME))
env.terminate()
return
# Manager code.
mprint('Loading fix samples')
with open(FIXED_SAMPLES_FILENAME, 'rb') as fixed_samples_f:
fix_samples = pickle.load(fixed_samples_f)
evaluation_dirs = set()
play_dirs = set()
save_model(saver, sess, args.ai_input_dir, epsilon=1.0)
epsilon_generator = LinearDecayGreedyEpsilonPolicy(
1.0, args.epsilon, TOTAL_WORKER_JOBS / 5.0)
fits_so_far = 0
mprint('Begin to train (now safe to run gcloud)')
mprint('Initial mean_max_q: ' + str(calculate_mean_max_Q(sess, online_model, fix_samples)))
while len(play_dirs) < TOTAL_WORKER_JOBS:
output_dirs = os.listdir(args.ai_output_dir)
output_dirs = [os.path.join(args.ai_output_dir, x) for x in output_dirs]
output_dirs = set(x for x in output_dirs if os.path.isdir(x))
new_dirs = sorted(output_dirs - evaluation_dirs - play_dirs)
if len(new_dirs) == 0:
time.sleep(0.1)
continue
new_dir = new_dirs[-1] # Most recent gameplay.
evaluation_path = os.path.join(new_dir, WORKER_OUTPUT_EVALUATE_FILENAME)
if os.path.isfile(evaluation_path):
evaluation_dirs.add(new_dir)
with open(evaluation_path, 'rb') as evaluation_file:
rewards, game_lengths, mean_max_Q = pickle.load(evaluation_file)
evaluation = [np.mean(rewards), np.std(rewards),
np.mean(game_lengths), np.std(game_lengths),
mean_max_Q]
mprint('Evaluation: ' + '\t'.join(str(x) for x in evaluation))
continue
memory_path = os.path.join(new_dir, WORKER_OUTPUT_GAMEPLAY_FILENAME)
try:
if os.path.getsize(memory_path) == 0:
# TODO Figure out why this happens despite temporary directory work.
# Also sometimes the file doesn't exist? Hence the try/except.
mprint('Output not ready somehow: ' + memory_path)
time.sleep(0.1)
continue
with open(memory_path, 'rb') as memory_file:
worker_memories = pickle.load(memory_file)
except Exception as exception:
print('Error reading ' + memory_path + ': ' + str(exception.args))
time.sleep(0.1)
continue
for worker_memory in worker_memories:
replay_memory.append(*worker_memory)
if args.psc:
os.remove(memory_path)
play_dirs.add(new_dir)
if len(play_dirs) <= NUM_BURN_IN_JOBS:
mprint('Skip training because still burn in.')
mprint('len(worker_memories): ' + str(len(worker_memories)))
continue
for _ in range(int(len(worker_memories) * FITS_PER_SINGLE_MEMORY)):
agent.fit(sess, fits_so_far)
fits_so_far += 1
# Partial evaluation to give frequent insight into agent progress.
# Last time checked, this took ~0.1 seconds to complete.
mprint('mean_max_q, len(worker_memories): ' +
str(calculate_mean_max_Q(sess, online_model, fix_samples)) +
', ' + str(len(worker_memories)))
# Always decrement epsilon (e.g. not just when saving model).
model_epsilon = epsilon_generator.get_epsilon(decay_epsilon=True)
if len(play_dirs) % SAVE_MODEL_EVERY == 0:
save_model(saver, sess, args.ai_input_dir, model_epsilon)
BATCH_SIZE = 32
gamma = 0.99
epsilon = 0.05
C = 10000
env = gym.make("Breakout-v0")
#number of actions, used to construct an policy selector
num_actions = env.action_space.n
#create helpers
#observation processor
atari_processor = AtariProcessor(IMAGE_SIZE)
history_store = HistoryStore(HISTORY_LENGTH, IMAGE_SIZE)
#policy selector, for testing, use uniform random policy selector, just pass number of actions to the constructor
random_selector = UniformRandomPolicy(num_actions)
greedy_selector = GreedyPolicy()
greedy_epsilon_selector = GreedyEpsilonPolicy(epsilon)
greedy_epsilon_linear_decay_selector = LinearDecayGreedyEpsilonPolicy(
1, 0.05, int(round(MAX_INTERACTION / 5, 0)))
# Initialize neural network
# Online network which changes during training but not to calculate Q*.
model_online = NN_cnn((IMAGE_SIZE[0], IMAGE_SIZE[1], HISTORY_LENGTH),
num_actions)
# Fixed network which is not changed during training but to calculate Q*.
model_fixed = NN_cnn((IMAGE_SIZE[0], IMAGE_SIZE[1], HISTORY_LENGTH),
num_actions)
model_fixed.model.set_weights(model_online.model.get_weights())
#model_fixed.model = Model.from_config(model_online.model.get_config())
def main():
#env = gym.make("Enduro-v0")
#env = gym.make("SpaceInvaders-v0")
#env = gym.make("Breakout-v0")
model_name = "result-q6-qqdn"
if (len(sys.argv) >= 2):
model_name = sys.argv[1]
if (len(sys.argv) >= 3):
env = gym.make(sys.argv[2])
else:
#env = gym.make("Enduro-v0")
env = gym.make("SpaceInvaders-v0")
#env = gym.make("Breakout-v0")
#no skip frames
env.frameskip = 1
input_shape = (84, 84)
batch_size = 32
num_actions = env.action_space.n
memory_size = 1000000
memory_burn_in_num = 50000
start_epsilon = 1
end_epsilon = 0.01
decay_steps = 1000000
target_update_freq = 10000
train_freq = 4 #How often you train the network
history_size = 4
history_prep = HistoryPreprocessor(history_size)
atari_prep = AtariPreprocessor(input_shape, 0, 999)
numpy_prep = NumpyPreprocessor()
preprocessors = PreprocessorSequence(
[atari_prep, history_prep, numpy_prep]) #from left to right
policy = LinearDecayGreedyEpsilonPolicy(start_epsilon, end_epsilon,
decay_steps)
model = create_model(history_size, input_shape, num_actions, model_name)
model.summary()
#plot_model(model,to_file="dueling.png")
optimizer = Adam(lr=0.001,
beta_1=0.9,
beta_2=0.999,
epsilon=1e-08,
decay=0.0)
loss_func = huber_loss
#linear_model.compile(optimizer, loss_func)
random_policy = UniformRandomPolicy(num_actions)
#memory = ActionReplayMemory(1000000,4)
memory = ActionReplayMemory(memory_size, 4)
memory_burn_in(env, memory, preprocessors, memory_burn_in_num,
random_policy)
#print(reward_arr)
#print(curr_state_arr)
agent = DDQNAgent(model, preprocessors, memory, policy, 0.99,
target_update_freq, None, train_freq, batch_size)
agent.compile(optimizer, loss_func)
agent.save_models()
agent.fit(env, 1000000, 100000)