def testEvalOnce(self):
pe = PolicyMonitor(env=self.env,
policy_net=self.global_policy_net,
summary_writer=self.summary_writer)
with self.test_session() as sess:
sess.run(tf.initialize_all_variables())
total_reward, episode_length = pe.eval_once(sess)
self.assertTrue(episode_length > 0)
def testEvalOnce(self):
pe = PolicyMonitor(
env=self.env,
policy_net=self.global_policy_net,
summary_writer=self.summary_writer)
with self.test_session() as sess:
sess.run(tf.initialize_all_variables())
total_reward, episode_length = pe.eval_once(sess)
self.assertTrue(episode_length > 0)
worker_summary_writer = summary_writer
worker = Worker(name="worker_{}".format(worker_id),
env=make_env(),
policy_net=policy_net,
value_net=value_net,
global_counter=global_counter,
discount_factor=0.99,
summary_writer=worker_summary_writer,
max_global_steps=FLAGS.max_global_steps)
workers.append(worker)
saver = tf.train.Saver(keep_checkpoint_every_n_hours=2.0, max_to_keep=10)
pe = PolicyMonitor(env=make_env(wrap=False),
policy_net=policy_net,
summary_writer=summary_writer,
saver=saver)
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
coord = tf.train.Coordinator()
latest_checkpoint = tf.train.latest_checkpoint(CHECKPOINT_DIR)
if latest_checkpoint:
print("Loading model checkpoint: {}".format(latest_checkpoint))
saver.restore(sess, latest_checkpoint)
worker_threads = []
for worker in workers:
# Session configs
if FLAGS.gpuid is not None:
os.environ["CUDA_VISIBLE_DEVICES"] = FLAGS.gpuid
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
with tf.device("/cpu:0"):
saver = tf.train.Saver(keep_checkpoint_every_n_hours=1.0, max_to_keep=1)
# Used to occasionally write episode rewards to Tensorboard
pe = PolicyMonitor(envs=make_envs(),
policy_net=policy_net,
domain=FLAGS.domain,
instances=instances,
neighbourhood=FLAGS.neighbourhood,
summary_writer=val_summary_writer,
saver=saver)
results = {}
with tf.Session(config=config) as sess:
sess.run(tf.global_variables_initializer())
coord = tf.train.Coordinator()
for file in os.listdir(FLAGS.restore_dir + "/checkpoints"):
print(file)
if file[-4:] == "meta":
file_num = int(file[6:-5])
env=make_env(),
policy_net=policy_net,
value_net=value_net,
global_counter=global_counter,
discount_factor = 0.99,
summary_writer=worker_summary_writer,
max_global_steps=FLAGS.max_global_steps)
workers.append(worker)
saver = tf.train.Saver(keep_checkpoint_every_n_hours=2.0, max_to_keep=10)
# Used to occasionally save videos for our policy net
# and write episode rewards to Tensorboard
pe = PolicyMonitor(
env=make_env(),
policy_net=policy_net,
summary_writer=summary_writer,
saver=saver)
with tf.Session() as sess:
sess.run(tf.initialize_all_variables())
coord = tf.train.Coordinator()
# Load a previous checkpoint if it exists
latest_checkpoint = tf.train.latest_checkpoint(CHECKPOINT_DIR)
if latest_checkpoint:
print("Loading model checkpoint: {}".format(latest_checkpoint))
saver.restore(sess, latest_checkpoint)
# Start worker threads
worker_threads = []
def main():
# Depending on the game we may have a limited action space
env_ = gym.make(FLAGS.env)
num_actions = env_.action_space.n
dim_obs = list(env_.observation_space.shape)
assert len(dim_obs) == 3 and dim_obs[2] == 3 #make sure it is a RGB frame
N_FRAME = FLAGS.n_frame if FLAGS.n_frame else 1
dim_obs[2] *= N_FRAME
print("Valid number of actions is {}".format(num_actions))
print("The dimension of the observation space is {}".format(dim_obs))
env_.close()
# Set the number of workers
NUM_WORKERS = (FLAGS.parallelism
if FLAGS.parallelism else multiprocessing.cpu_count())
MODEL_DIR = FLAGS.model_dir
CP_H = FLAGS.checkpoint_hour
CHECKPOINT_DIR = os.path.join(MODEL_DIR, "checkpoints")
TENSORBOARD_DIR = os.path.join(MODEL_DIR, "tb")
# Optionally empty model directory
if FLAGS.reset:
shutil.rmtree(MODEL_DIR, ignore_errors=True)
if not os.path.exists(CHECKPOINT_DIR):
os.makedirs(CHECKPOINT_DIR)
summary_writer = tf.summary.FileWriter(TENSORBOARD_DIR)
with tf.device("/cpu:0"):
# Keeps track of the number of updates we've performed
global_step = tf.Variable(0, name="global_step", trainable=False)
# Global policy and value nets
with tf.variable_scope("global") as vs:
policy_net = PolicyEstimator(num_outputs=num_actions,
dim_inputs=dim_obs)
value_net = ValueEstimator(reuse=True, dim_inputs=dim_obs)
# Global step iterator
global_counter = itertools.count()
# Create worker graphs
workers = []
for worker_id in range(NUM_WORKERS):
# We only write summaries in one of the workers because they're
# pretty much identical and writing them on all workers
# would be a waste of space
worker_summary_writer = None
if worker_id == 0:
worker_summary_writer = summary_writer
worker = Worker(name="worker_{}".format(worker_id),
env=gym.make(FLAGS.env),
policy_net=policy_net,
value_net=value_net,
global_counter=global_counter,
discount_factor=0.99,
summary_writer=worker_summary_writer,
max_global_steps=FLAGS.max_global_steps,
n_frame=N_FRAME)
workers.append(worker)
saver = tf.train.Saver(keep_checkpoint_every_n_hours=CP_H,
max_to_keep=10)
# Used to occasionally save videos for our policy net
# and write episode rewards to Tensorboard
pe = PolicyMonitor(env=gym.make(FLAGS.env),
policy_net=policy_net,
summary_writer=summary_writer,
saver=saver)
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
coord = tf.train.Coordinator()
# Load a previous checkpoint if it exists
latest_checkpoint = tf.train.latest_checkpoint(CHECKPOINT_DIR)
if latest_checkpoint:
print("Loading model checkpoint: {}".format(latest_checkpoint))
saver.restore(sess, latest_checkpoint)
# Start worker threads
worker_threads = []
for worker in workers:
print("starting worker:")
worker_fn = lambda: worker.run(sess, coord, FLAGS.t_max)
t = threading.Thread(target=worker_fn)
t.start()
worker_threads.append(t)
# Start a thread for policy eval task
monitor_thread = threading.Thread(
target=lambda: pe.continuous_eval(FLAGS.eval_every, sess, coord))
monitor_thread.start()
# Wait for all workers to finish
coord.join(worker_threads)
domain=FLAGS.domain,
instances=instances,
N_train=N_train_instances,
neighbourhood=FLAGS.neighbourhood,
discount_factor=0.99,
summary_writer=worker_summary_writer,
max_global_steps=FLAGS.max_global_steps)
workers.append(worker)
saver = tf.train.Saver(keep_checkpoint_every_n_hours=0.25)
# Used to occasionally write episode rewards to Tensorboard
pe = PolicyMonitor(envs=make_envs(),
policy_net=policy_net,
domain=FLAGS.domain,
instances=instances,
neighbourhood=FLAGS.neighbourhood,
summary_writer=val_summary_writer,
saver=saver)
# print([n.name for n in tf.get_default_graph().as_graph_def().node])
with tf.Session(config=config) as sess:
sess.run(tf.global_variables_initializer())
# train_summary_writer.add_graph(sess.graph)
# val_summary_writer.add_graph(sess.graph)
coord = tf.train.Coordinator()
if FLAGS.use_pretrained:
# Load a previous checkpoint if it exists
ckpt = tf.train.get_checkpoint_state(
env=make_env(),
policy_net=policy_net,
value_net=value_net,
global_counter=global_counter,
discount_factor = 0.99,
summary_writer=worker_summary_writer,
max_global_steps=FLAGS.max_global_steps)
workers.append(worker)
saver = tf.train.Saver(keep_checkpoint_every_n_hours=2.0, max_to_keep=10)
# Used to occasionally save videos for our policy net
# and write episode rewards to Tensorboard
pe = PolicyMonitor(
env=make_env(wrap=False),
policy_net=policy_net,
summary_writer=summary_writer,
saver=saver)
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
coord = tf.train.Coordinator()
# Load a previous checkpoint if it exists
latest_checkpoint = tf.train.latest_checkpoint(CHECKPOINT_DIR)
if latest_checkpoint:
print("Loading model checkpoint: {}".format(latest_checkpoint))
saver.restore(sess, latest_checkpoint)
# Start worker threads
worker_threads = []
worker = Worker(name="worker_{}".format(worker_id),
env=make_env(thread_name=str(worker_id)),
policy_net=policy_net,
value_net=value_net,
global_counter=global_counter,
discount_factor=0.99,
summary_writer=worker_summary_writer,
max_global_steps=FLAGS.max_global_steps)
workers.append(worker)
saver = tf.train.Saver(keep_checkpoint_every_n_hours=2.0, max_to_keep=10)
# Used to occasionally save videos for our policy net
# and write episode rewards to Tensorboard
pe = PolicyMonitor(env=make_env(thread_name=str("p")),
policy_net=policy_net,
summary_writer=summary_writer,
saver=saver)
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
coord = tf.train.Coordinator()
# Load a previous checkpoint if it exists
latest_checkpoint = tf.train.latest_checkpoint(CHECKPOINT_DIR)
if latest_checkpoint:
print("Loading model checkpoint: {}".format(latest_checkpoint))
saver.restore(sess, latest_checkpoint)
# Start worker threads
worker_threads = []
for worker in workers:
worker = Worker(name="worker_{}".format(worker_id),
env=getEnv(FLAGS, PORT + worker_id + 1),
policy_net=policy_net,
value_net=value_net,
global_counter=global_counter,
discount_factor=0.99,
summary_writer=worker_summary_writer,
max_global_steps=FLAGS.max_global_steps)
workers.append(worker)
saver = tf.train.Saver(keep_checkpoint_every_n_hours=2.0, max_to_keep=10)
# Used to occasionally save videos for our policy net
# and write episode rewards to Tensorboard
pe = PolicyMonitor(env=_env,
policy_net=policy_net,
summary_writer=summary_writer,
saver=saver)
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
coord = tf.train.Coordinator()
# Load a previous checkpoint if it exists
latest_checkpoint = tf.train.latest_checkpoint(CHECKPOINT_DIR)
if latest_checkpoint:
print("Loading model checkpoint: {}".format(latest_checkpoint))
saver.restore(sess, latest_checkpoint)
# Start worker threads
worker_threads = []
for worker in workers: