def train_MetaNetwork():
print("\n ---- Training the Meta Network ----- \n")
MODEL_NAME = "meta_grid16_zero_2"
MODEL_NAME_save = "meta_grid16_zero_2"
DIAMOND_MODEL_NAME = "diamond_grid16_4"
ZOMBIE_MODEL_NAME = "zombie_grid16_2"
EXPLORE_MODEL_NAME = "explore_grid16_2"
# EXTRA_MODEL_NAME = "extra15_input6_2"
# MODEL_NAME = "meta15_input6_1M_unfrozen_dojos"
# DIAMOND_MODEL_NAME = "diamond15_input4_best_unfrozen_at_1M"
# ZOMBIE_MODEL_NAME = "zombie15_input4_best_unfrozen_at_1M"
# EXPLORE_MODEL_NAME = "explore15_input4_best_unfrozen_at_1M"
# MODEL_NAME = "meta15_input6_1M_random_unfrozen_cointoss"
# DIAMOND_MODEL_NAME = "diamond15_input4_1M_random_unfrozen_cointoss"
# ZOMBIE_MODEL_NAME = "zombie15_input4_1M_random_unfrozen_cointoss"
# EXPLORE_MODEL_NAME = "explore15_input4_1M_random_unfrozen_cointoss"k
FOLDER = "Impossible"
DOJO_FOLDER = "Impossible"
MODEL_PATH_SAVE = "./Models/Tensorflow/"+FOLDER+"/"+MODEL_NAME+"/"+MODEL_NAME+".ckpt"
LOGDIR = "./Logs/"+FOLDER+"/"+MODEL_NAME_save+""
USE_SAVED_MODEL_FILE = False
GRID_SIZE = 16
LOCAL_GRID_SIZE = 15
MAP_PATH = None
RANDOMIZE_MAPS = True
RENDER_TO_SCREEN = False
# RENDER_TO_SCREEN = True
env = Environment(wrap = False,
grid_size = GRID_SIZE,
local_size = LOCAL_GRID_SIZE,
rate = 80,
max_time = 120,
food_count = 0,
obstacle_count = 0,
lava_count = 0,
zombie_count = 0,
history = 100,
action_space = 5,
map_path = MAP_PATH)
if RENDER_TO_SCREEN:
env.prerender()
model = MetaNetwork(local_size=LOCAL_GRID_SIZE, name=MODEL_NAME, path="./Models/Tensorflow/"+FOLDER+"/", load=False, trainable=True)
diamond_net = Network(local_size=LOCAL_GRID_SIZE, name=DIAMOND_MODEL_NAME, path="./Models/Tensorflow/"+DOJO_FOLDER+"/", load=True, trainable=False)
zombie_net = Network(local_size=LOCAL_GRID_SIZE, name=ZOMBIE_MODEL_NAME, path="./Models/Tensorflow/"+DOJO_FOLDER+"/", load=True, trainable=False)
explore_net = Network(local_size=LOCAL_GRID_SIZE, name=EXPLORE_MODEL_NAME, path="./Models/Tensorflow/"+DOJO_FOLDER+"/", load=True, trainable=False)
# extra_net = Network(local_size=LOCAL_GRID_SIZE, name=EXTRA_MODEL_NAME, path="./Models/Tensorflow/"+DOJO_FOLDER+"/", load=False, trainable=True)
brain = Brain(epsilon=0.05, action_space=3)
model.setup(brain)
diamond_net.setup(brain)
zombie_net.setup(brain)
explore_net.setup(brain)
# extra_net.setup(brain)
score = tf.placeholder(tf.float32, [])
avg_t = tf.placeholder(tf.float32, [])
epsilon = tf.placeholder(tf.float32, [])
avg_r = tf.placeholder(tf.float32, [])
tf.summary.scalar('error', tf.squeeze(model.error))
tf.summary.scalar('score', score)
tf.summary.scalar('average time', avg_t)
tf.summary.scalar('epsilon', epsilon)
tf.summary.scalar('avg reward', avg_r)
avg_time = 0
avg_score = 0
avg_error = 0
avg_reward = 0
cumulative_reward = 0
# Number of episodes
print_episode = 1000
total_episodes = 100000
saver = tf.train.Saver()
# Initialising all variables (weights and biases)
init = tf.global_variables_initializer()
# Adds a summary graph of the error over time
merged_summary = tf.summary.merge_all()
# Tensorboard capabilties
writer = tf.summary.FileWriter(LOGDIR)
# Histogram
histogram = Histogram(3, 10, total_episodes)
# GPU capabilities
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.3)
# Begin session
with tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) as sess:
if USE_SAVED_MODEL_FILE:
saver.restore(sess, MODEL_PATH_SAVE)
print("Model restored.")
else:
sess.run(init)
writer.add_graph(sess.graph)
start_time = time.time()
print("")
for episode in range(total_episodes):
if RANDOMIZE_MAPS:
# Make a random map 0: lava, 1: obstacle
MAP_PATH = "./Maps/Grid{}/impossible_map{}.txt".format(GRID_SIZE, np.random.randint(5))
env.set_map(MAP_PATH)
# state, info = env.reset()
state, info = env.quick_reset()
done = False
# brain.linear_epsilon_decay(total_episodes, episode, start=1.0, end=0.05, percentage=0.5)
# brain.linear_alpha_decay(total_episodes, episode)
if RENDER_TO_SCREEN:
env.render()
while not done:
# Retrieve the Q values from the NN in vector form
Dojo_vector = sess.run(model.q_values, feed_dict={model.input: state})
dojo = brain.choose_action(state, sess, model)
histogram.check_section(episode)
histogram.add(dojo)
# dojo = np.random.randint(3)
# dojo = 0
# print(dojo)
if dojo == 0:
dojo_state = state
# dojo_state[2]=0
# dojo_state[3]=0
# dojo_state = np.delete(dojo_state, 2, 0)# Take out the zombie layer
# dojo_state = np.delete(dojo_state, 2, 0)# Take out the history layer
action = brain.choose_dojo(dojo_state, sess, diamond_net, env.number_of_actions(), 0.05)
elif dojo == 1:
dojo_state = state
# dojo_state[1]=0
# dojo_state[3]=0
# dojo_state = np.delete(dojo_state, 1, 0)# Take out the diamond layer
# dojo_state = np.delete(dojo_state, 2, 0)# Take out the history layer
action = brain.choose_dojo(dojo_state, sess, zombie_net, env.number_of_actions(), 0.05)
elif dojo == 2:
dojo_state = state
# dojo_state[1]=0
# dojo_state[2]=0
# dojo_state = np.delete(dojo_state, 1, 0)# Take out the diamond layer
# dojo_state = np.delete(dojo_state, 1, 0)# Take out the zombie layer
action = brain.choose_dojo(dojo_state, sess, explore_net, env.number_of_actions(), 0.05)
# elif dojo == 3:
# dojo_state = state
# action = brain.choose_dojo(dojo_state, sess, extra_net, env.number_of_actions(), 0.05)
# print(action)
# Update environment with by performing action
new_state, reward, done, info = env.step(action)
# print(new_state)
brain.store_transition_dojo(state, action, reward, done, new_state, dojo)
# print(tf.trainable_variables(scope=None))
# if dojo == 0:
# e, Q_vector = brain.train_3_dojos(diamond_net, sess, dojo)
# elif dojo == 1:
# e, Q_vector = brain.train_3_dojos(zombie_net, sess, dojo)
# elif dojo == 2:
# e, Q_vector = brain.train_3_dojos(explore_net, sess, dojo)
# e, Q_vector = brain.train_3(sess, diamond_net, zombie_net, explore_net)
# e, Q_vector = brain.train(extra_net, sess)
if done:
Dojo_vector[:,dojo] = reward
# print("Reward:", reward)
else:
# Gathering the now current state's action-value vector
y_prime = sess.run(model.q_values, feed_dict={model.input: new_state})
# Equation for training
maxq = sess.run(model.y_prime_max, feed_dict={model.actions: y_prime})
# RL Equation
Dojo_vector[:,dojo] = reward + (brain.GAMMA * maxq)
_, e = sess.run([model.optimizer, model.error], feed_dict={model.input: state, model.actions: Dojo_vector})
state = new_state
cumulative_reward += reward
if RENDER_TO_SCREEN:
env.render()
if done:
avg_time += info["time"]
avg_score += info["score"]
avg_error += e
avg_reward += cumulative_reward
cumulative_reward = 0
if (episode%print_episode == 0 and episode != 0) or (episode == total_episodes-1):
current_time = math.floor(time.time()-start_time)
print("Ep:", episode,
"\tavg t: {0:.3f}".format(avg_time/print_episode),
"\tavg score: {0:.3f}".format(avg_score/print_episode),
"\tErr {0:.3f}".format(avg_error/print_episode),
"\tavg_reward {0:.3f}".format(avg_reward/print_episode), # avg cumulative reward
"\tepsilon {0:.3f}".format(brain.EPSILON),
end="")
print_readable_time(current_time)
# Save the model's weights and biases to .npz file
model.save(sess, name=MODEL_NAME_save)
# diamond_net.save(sess, name=DIAMOND_MODEL_NAME+"")
# zombie_net.save(sess, name=ZOMBIE_MODEL_NAME+"")
# explore_net.save(sess, name=EXPLORE_MODEL_NAME+"")
# extra_net.save(sess, name=EXTRA_MODEL_NAME+"")
# save_path = saver.save(sess, MODEL_PATH_SAVE)
s = sess.run(merged_summary, feed_dict={model.input: state, model.actions: Dojo_vector, score:avg_score/print_episode, avg_t:avg_time/print_episode, epsilon:brain.EPSILON, avg_r:avg_reward/print_episode})
writer.add_summary(s, episode)
avg_time = 0
avg_score = 0
avg_error = 0
avg_reward = 0
model.save(sess, verbose=True, name=MODEL_NAME_save)
# diamond_net.save(sess, verbose=True, name=DIAMOND_MODEL_NAME+"")
# zombie_net.save(sess, verbose=True, name=ZOMBIE_MODEL_NAME+"")
# explore_net.save(sess, verbose=True, name=EXPLORE_MODEL_NAME+"")
# extra_net.save(sess, verbose=True, name=EXTRA_MODEL_NAME+"")
# save_path = saver.save(sess, MODEL_PATH_SAVE)
# print("Model saved in path: %s" % save_path)
writer.close()
histogram.plot()
import sys
from utils import custom_epsilon
import matplotlib.pyplot as plt
# import getch
import csv
import my_mission
from utils import print_readable_time, Histogram
histogram = Histogram(3, 4, 1000)
for i in range(1000):
histogram.add(np.random.randint(3))
histogram.plot()
# print(my_mission.missionXML)
# s = np.array([[[0,0],
# [0,0]],
# [[1,1],
# [1,1]],
# [[2,2],
# [2,2]]])
# s[0] = 1
# print(s)
# t = []