def testing(self):
from keepitpossible.common import action_table
self.table_action = action_table.create_action_table()
self.MODEL.load()
done = False
reward = 0.0
env = ObstacleTowerEnv(environment_filename=self.SCHEDULE.ENV_PATH,
worker_id=self.SCHEDULE.N_WORKER + 1,
retro=False,
realtime_mode=True)
obs = env.reset()
previous_preprocessed_observation_image = obs[0]
while not done:
action = self.MODEL.choose_action(
previous_preprocessed_observation_image)
# 做出動作，獲得場景資訊,已過關數,代理資訊
for _ in self.table_action[int(action)]:
observation, reward, done, info = env.step(_)
print(
"Action_Chose: ",
action,
"Action: ",
_,
" Reward: ",
reward)
if done:
break
# 預處理模型需要的資料
observation_image, keys, time_remaining = observation
preprocessed_observation_image = observation_image
previous_preprocessed_observation_image = preprocessed_observation_image
env.close()
def seed_hashes():
mapping = {}
while len(mapping) < 100:
if os.path.exists('UnitySDK.log'):
os.remove('UnitySDK.log')
while True:
try:
env = ObstacleTowerEnv(os.environ['OBS_TOWER_PATH'],
worker_id=random.randrange(1000))
break
except KeyboardInterrupt:
sys.exit(1)
except:
pass
env.seed(25) # random argument
obs = env.reset()
env.close()
with open('UnitySDK.log') as f:
contents = next(l for l in f.readlines() if 'seed:' in l)
seed = int(contents.split(': ')[-1])
yield str(obs.flatten().tolist()), seed
return mapping
def main():
if len(sys.argv) != 2:
sys.stderr.write('Usage: record_improve.py <recording_path>\n')
os.exit(1)
rec = Recording(sys.argv[1])
env = ObstacleTowerEnv(os.environ['OBS_TOWER_PATH'],
worker_id=random.randrange(11, 20))
try:
env.seed(rec.seed)
if rec.floor:
env.floor(rec.floor)
env.reset()
i = 0
for i, (action, rew) in enumerate(zip(rec.actions, rec.rewards)):
_, real_rew, done, _ = env.step(action)
if not np.allclose(real_rew, rew):
print('mismatching result at step %d' % i)
sys.exit(1)
if done != (i == rec.num_steps - 1):
print('invalid done result at step %d' % i)
sys.exit(1)
print('match succeeded')
finally:
env.close()
18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
18, 18, 24, 24, 24, 18, 18, 18, 18, 30, 18, 18, 30, 30, 30, 30, 30, 30,
30, 24, 24, 24, 24, 6, 18, 24, 24, 24, 24, 6, 6, 6, 6, 6, 24, 18, 18,
18, 18, 30, 18, 18, 18, 18, 18, 18, 18, 18, 30, 30, 30, 18, 18, 24, 24,
24, 6, 6, 6, 6, 6, 6, 6, 18, 18, 18, 18, 24, 24, 18, 18, 30, 30, 18,
18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 24, 30,
18, 18, 18, 30, 30, 30, 18, 18, 18, 18, 30, 18, 24, 24, 24, 24, 24, 18,
24, 30, 18, 18, 18, 18, 24, 30, 30, 30, 30, 30, 30, 30, 30, 24, 24, 24,
6, 6, 6, 6, 6, 24, 24, 18, 18, 18, 24, 18, 18, 18, 18, 18, 18, 18, 18,
18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 24, 18, 18, 18, 18, 30, 18,
24, 24, 18, 18, 30, 30, 18, 24, 24, 24, 30, 30, 30, 18, 18, 18, 18, 18,
18, 18, 18, 18, 18, 24, 18, 18, 18, 18, 24, 24, 18, 18, 30, 24, 6, 6,
6, 6, 6, 6, 24, 24, 24, 18, 18, 30, 30, 30, 18, 18, 24, 24, 24, 18, 18,
30, 30, 30, 30, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 18,
18, 18, 18, 18, 18, 18, 18, 18, 24, 24, 21, 30, 12, 30, 18, 18, 18, 18,
30, 30, 18, 18, 18, 18, 24, 30, 30, 30, 30, 18, 18, 24, 24, 24, 24, 6,
6, 6
]:
env.step(action)
env.seed(37)
env.floor(11)
print('resetting (this will hang)...')
env.reset()
print('done reset')
env.close()
class Worker(threading.Thread):
episode_count = 0
mean_reward = 0
best_score = 0
global_steps = 0
save_lock = threading.Lock()
def __init__(self, result_queue, idx, save_dir, params):
super(Worker, self).__init__()
self.result_queue = result_queue
self.worker_idx = idx
self.save_dir = save_dir
self.model_path = os.path.join(self.save_dir, 'model_a3c')
self.env = ObstacleTowerEnv(params['env_path'],
worker_id=self.worker_idx,
retro=False,
realtime_mode=False,
greyscale=False,
config=train_env_reset_config)
self.action_size = params['action_size']
self._action_lookup = params['action_lookup']
self.input_shape = self.env.observation_space[0].shape # (84, 84, 3)
self._last_health = 99999.
self._last_keys = 0
self.global_model = params['global_model']
# self.local_model = CNN(self.action_size, self.input_shape)
self.local_model = CnnGru(self.action_size, self.input_shape)
self.ac_ckpt = params['ckpt']
self.ac_manager = params['ckpt_mgr']
self.current_time = params['log_timestamp']
train_log_dir = './logs/' + self.current_time + '/worker_' + str(
self.worker_idx)
self.worker_summary_writer = tf.summary.create_file_writer(
train_log_dir)
self.timesteps = params['timesteps']
self.batch_size = params['batch_size']
self.gamma = params['gamma']
self.lr = params['lr']
self.opt = params['optimizer']
self.eps = np.finfo(np.float32).eps.item()
def get_updated_reward(self, reward, new_health, new_keys, done):
new_health = float(new_health)
new_reward = 0.0
if done: # reset params when game is terminated
self._last_health = 99999.
self._last_keys = 0
else:
# opened a door, solved a puzzle, picked up a key
if 0.1 <= reward < 1:
new_reward += 0.5
# crossing a floor - between [1, 4]
if reward >= 1:
new_reward += (new_health / 10000)
# found time orb / crossed a floor
if new_health > self._last_health:
new_reward += 0.5
return new_reward
def log_worker_metrics(self, episode_reward, loss, step):
with self.worker_summary_writer.as_default():
with tf.name_scope('worker'):
tf.summary.scalar('reward', episode_reward, step=step)
tf.summary.scalar('loss', loss, step=step)
self.worker_summary_writer.flush()
def run(self):
mem = Memory()
ep_count = 0
timestep = 0
entropy_term = 0
ep_reward = 0.
ep_steps = 0
ep_loss = 0.
done = False
obs = self.env.reset()
state, self._last_keys, self._last_health, _ = obs
while timestep <= self.timesteps:
i = 0
with tf.GradientTape() as tape:
while i < self.batch_size:
# collect experience
# get action as per policy
state = tf.convert_to_tensor(state)
state = tf.expand_dims(state, axis=0)
action_probs, critic_value = self.local_model(
[state, float(self._last_health)], training=True)
entropy = -np.sum(action_probs * np.log(action_probs))
entropy_term += entropy
# choose most probable action
dist = tfp.distributions.Categorical(probs=action_probs,
dtype=tf.float32)
action_index = int(dist.sample().numpy())
action = self._action_lookup[action_index]
# perform action in game env
for i in range(4): # frame skipping
obs, reward, done, _ = self.env.step(action)
state, new_keys, new_health, cur_floor = obs
reward = self.get_updated_reward(
reward, new_health, new_keys, done)
self._last_health = new_health
self._last_keys = new_keys
ep_reward += reward
ep_steps += 1
i += 1
timestep += 1
# store experience
mem.store(action_prob=tf.math.log(
action_probs[0, action_index]),
value=critic_value[0, 0],
reward=reward)
if done:
break
# backpropagation
total_loss = self.local_model.compute_loss(
mem, state, done, self.gamma, self.eps, entropy_term)
ep_loss += total_loss
Worker.global_steps += ep_steps
grads = tape.gradient(total_loss,
self.local_model.trainable_variables
) # calculate local gradients
self.opt.apply_gradients(
zip(grads, self.global_model.trainable_variables)
) # send local gradients to global model
self.local_model.set_weights(self.global_model.get_weights(
)) # update local model with new weights
mem.clear()
if done:
Worker.mean_reward = (Worker.mean_reward * Worker.episode_count
+ ep_reward) / (Worker.episode_count + 1)
self.log_worker_metrics(ep_reward, ep_loss, ep_count)
print(
"Episode: {} | Mean Reward: {:.3f} | Episode Reward: {:.3f} | Loss: {:.3f} | Steps: {} | Total Steps: {} | Worker: {}"
.format(Worker.episode_count, Worker.mean_reward,
ep_reward, ep_loss, ep_steps, Worker.global_steps,
self.worker_idx))
self.result_queue.put((Worker.mean_reward, total_loss))
Worker.episode_count += 1
ep_count += 1
obs = self.env.reset()
state, _, _, _ = obs
# use a lock to save local model and to print to prevent data races.
if ep_reward > Worker.best_score:
with Worker.save_lock:
self.ac_manager.save()
print("Saved checkpoint for step {}".format(
int(self.ac_ckpt.step)))
self.ac_ckpt.step.assign_add(1)
keras.models.save_model(self.global_model,
self.model_path)
print('\nSaved best model to: {}, episode score: {}\n'.
format(self.model_path, ep_reward))
Worker.best_score = ep_reward
entropy_term = 0
ep_reward = 0.
ep_steps = 0
ep_loss = 0.
self.result_queue.put(None)
self.env.close()
class Worker(object):
def __init__(self,
envpath,
wid,
retro,
realtime_mode,
env_seed=0,
env_floor=0):
self.wid = wid
self.env = ObstacleTowerEnv(environment_filename=envpath,
worker_id=wid,
retro=retro,
realtime_mode=realtime_mode)
self.kprun = GLOBAL_KPRUN
self.tableAction = self.createActionTable()
# 設定關卡
self.env_seed = env_seed
self.env_floor = env_floor
self.step = 0
self.summary = tf.Summary(value=[
tf.Summary.Value(tag="Stage_reward " + str(self.wid),
simple_value=0)
])
self.kprun.train_writer.add_summary(self.summary, 0)
def createActionTable(self):
tableAction = []
for a in range(0, 3):
for b in range(0, 3):
for c in range(0, 2):
tableAction.append([a, b, c, 0])
# print("Action option: ", tableAction[0:17])
return tableAction
def reward_compute(self, done, reward_total, keys, previous_keys, reward,
previous_reward, time_remaining,
previous_time_remaining, previous_stage_time_remaining):
# 定義獎勵公式
# reward 是從環境傳來的破關數
# keys 是撿到鑰匙的數量
# time_remaining 是剩餘時間
# 過關最大獎勵為10
# 一把鑰匙為5
# 時間果實暫時只給0.5，因為結束會結算剩餘時間，會有獎勵累加的問題。
# 如果過關，給予十倍過關獎勵 - (場景開始的時間-剩餘時間)/1000
# print("time_remaining ", time_remaining,
# " previous_time_remaining ", previous_time_remaining,
# " reward ", reward)
# 通過一個會開門的綠門會加0.1
if (reward - previous_reward) > 0 and (reward - previous_reward) < 0.3:
reward_total += 3
elif (reward - previous_reward) > 0.9:
# ***如果剩餘時間比場景時間多會變成加分獎勵，可能會極大增加Agent吃時間果實的機率。
# ***另一種方式是剩餘的時間直接/1000加上去，這樣就沒有累加效果。
print("Pass ", reward, " Stage!")
# reward_total += (reward - previous_reward) * 100 - \
# (previous_stage_time_remaining - time_remaining)
reward_total += 200
# 過關之後把時間留到下一關，儲存這回合時間供下次計算過關使用
previous_time_remaining = time_remaining
previous_stage_time_remaining = time_remaining
# Lesson 1 repeat
if reward > 6.5:
# self.total_step +=1
# if self.total_step >=5:
# done = True
# return reward_total, previous_stage_time_remaining, done
self.env.seed(np.random.randint(5))
# env.reset()
done = True
return reward_total, previous_stage_time_remaining, done
# 假設過關的時候有順便吃到果實或鑰匙，所以預設為同時可以加成
if previous_keys > keys:
# print("Get Key")
reward_total += 5
if previous_time_remaining < time_remaining and previous_time_remaining != 0:
# print("Get time power up")
reward_total += 2
else:
reward_total -= 0.5
if done and previous_time_remaining > 100:
print("Agent died")
# 如果剩餘時間越多就掛點，扣更多
# reward_total -= (10 + time_remaining / 100)
reward_total -= 100
return reward_total, previous_stage_time_remaining, done
def work(self):
global GLOBAL_EP, GLOBAL_RUNNING_R, GLOBAL_UPDATE_COUNTER
# 設定關卡
self.env.seed(self.env_seed)
self.env.floor(self.env_floor)
# 只要還沒達到目標回合就LOOP
while not COORD.should_stop():
# 紀錄步數
self.step += 1
# 重設關卡
obs = self.env.reset()
# 初始化
done = False
stage_reward = 0.0
reward = 0
keys = 0
# 檢查是否有吃到加時間的，如果是第一回合出來沒有time_remaining，事先定義
time_remaining = 3000
previous_stage_time_remaining = time_remaining
# 預處理圖像
# previous_preprocessed_observation_image = np.reshape(obs[0], [-1])
previous_preprocessed_observation_image = obs[0]
buffer_s, buffer_a, buffer_r = [], [], []
# 只要沒死
while not done:
# 如果模型正在更新就等待更新完成
if not ROLLING_EVENT.is_set():
# 等待更新完成
ROLLING_EVENT.wait()
# 清除記憶體，使用新的代理收集資料
buffer_s, buffer_a, buffer_r = [], [], []
# 儲存上一個動作狀態，供計算獎勵用
previous_keys = keys
previous_reward = reward
previous_time_remaining = time_remaining
# 根據上一次的狀態決定動作
action = self.kprun.choose_action(
previous_preprocessed_observation_image)
action = np.clip(np.random.normal(action, 1.), *[6, 12])
# 做出動作，獲得場景資訊,已過關數,代理資訊
observation, reward, done, info = self.env.step(
np.array(self.tableAction[int(action)]))
# 預處理模型需要的資料
observation_image, keys, time_remaining = observation
# preprocessed_observation_image = np.reshape(
# observation_image, [-1])
preprocessed_observation_image = observation_image
stage_reward, previous_stage_time_remaining, done = self.reward_compute(
done=done,
reward_total=stage_reward,
keys=keys,
previous_keys=previous_keys,
reward=reward,
previous_reward=previous_reward,
time_remaining=time_remaining,
previous_time_remaining=previous_time_remaining,
previous_stage_time_remaining=previous_stage_time_remaining
)
# Normalize reward~不知道中文怎麼打
stage_reward = stage_reward + 8 / 8
# 把這次狀態存入 記憶體
buffer_s.append(np.array([preprocessed_observation_image]))
buffer_a.append(action)
buffer_r.append(stage_reward)
# 儲存下一步要參考的圖像
previous_preprocessed_observation_image = preprocessed_observation_image
# 達到更新時，自己先做處理。
GLOBAL_UPDATE_COUNTER += 1
# 太多自己就先處理更新
if len(buffer_s) == EP_LEN - \
1 or GLOBAL_UPDATE_COUNTER >= MIN_BATCH_SIZE:
v_s_ = self.kprun.get_v(preprocessed_observation_image)
# 計算折扣獎勵
discounted_r = []
for r in buffer_r[::-1]:
v_s_ = r + GAMMA * v_s_
discounted_r.append(v_s_)
discounted_r.reverse()
# 整理維度
bs, ba, br = np.vstack(buffer_s), np.vstack(
buffer_a), np.array(discounted_r)[:, np.newaxis]
# 把資料放入共享記憶體
QUEUE.put(bs)
QUEUE.put(ba)
QUEUE.put(br)
# print("len(buffer_s)", len(buffer_s))
# print("bs.shape", bs.shape)
# 清空暫存
buffer_s, buffer_a, buffer_r = [], [], []
# 如果整個模型步數到達最小BATCH 就整個更新
if GLOBAL_UPDATE_COUNTER >= MIN_BATCH_SIZE:
# 停止收集資料
ROLLING_EVENT.clear()
# 更新PPO
UPDATE_EVENT.set()
# 達到最多EP停止訓練
if GLOBAL_EP >= EP_MAX:
COORD.request_stop()
break
# 紀錄獎勵
self.summary = tf.Summary(value=[
tf.Summary.Value(tag="Stage_reward " + str(self.wid),
simple_value=stage_reward)
])
self.kprun.train_writer.add_summary(self.summary, self.step)
GLOBAL_EP += 1
print(
'{0:.1f}%'.format(GLOBAL_EP / EP_MAX * 100),
'|W%i' % self.wid,
'|Ep_r: %.2f' % stage_reward,
)
self.env.close()
class RandomAgent:
"""Random Agent that will play the specified game
Args:
env_name: Name of the environment to be played
max_eps: Maximum number of episodes to run agent for.
"""
def __init__(self,
env_path,
train=False,
evaluate=False,
eval_seeds=[],
max_eps=100,
save_dir=None,
plot=False):
if train:
self.env = ObstacleTowerEnv(env_path,
worker_id=0,
retro=False,
realtime_mode=False,
config=train_env_reset_config)
else:
if evaluate:
self.env = ObstacleTowerEnv(env_path,
worker_id=0,
retro=False,
realtime_mode=False,
config=eval_env_reset_config)
self.env = ObstacleTowerEvaluation(self.env, eval_seeds)
else:
self.env = ObstacleTowerEnv(env_path,
worker_id=0,
retro=False,
realtime_mode=True,
config=eval_env_reset_config)
self.max_episodes = max_eps
self.global_moving_average_reward = 0
self.save_dir = save_dir
if not os.path.exists(save_dir):
os.makedirs(save_dir)
self.plot = plot
self.res_queue = Queue()
def train(self):
start_time = time.time()
reward_avg = 0
global_steps = 0
moving_average_rewards = []
for episode in range(self.max_episodes):
done = False
self.env.reset()
reward_sum = 0.0
steps = 0
while not done:
# Sample randomly from the action space and step
_, reward, done, _ = self.env.step(
self.env.action_space.sample())
steps += 1
global_steps += 1
reward_sum += reward
if self.plot:
# Record statistics
moving_average_rewards.append(reward_sum)
reward_avg += reward_sum
self.global_moving_average_reward = record(
episode, reward_sum, 0, self.global_moving_average_reward,
self.res_queue, 0, steps, global_steps)
end_time = time.time()
print("\nTraining complete. Time taken = {} secs".format(end_time -
start_time))
final_avg = reward_avg / float(self.max_episodes)
print("Average score across {} episodes: {}".format(
self.max_episodes, final_avg))
if self.plot:
plt.plot(moving_average_rewards)
plt.ylabel('Moving average episode reward')
plt.xlabel('Step')
plt.savefig(
os.path.join(self.save_dir, 'model_random_moving_average.png'))
self.env.close()
return final_avg
def play_single_episode(self):
action_space = ActionSpace()
print("Playing single episode...")
done = False
step_counter = 0
reward_sum = 0
obs = self.env.reset()
state, _, _, _ = obs
try:
while not done:
action = self.env.action_space.sample()
obs, reward, done, info = self.env.step(action)
reward_sum += reward
print("{}. Reward: {}, action: {}".format(
step_counter, reward_sum,
action_space.get_action_meaning(action)))
step_counter += 1
except KeyboardInterrupt:
print("Received Keyboard Interrupt. Shutting down.")
finally:
if not self.evaluate:
self.env.close()
return reward_sum
def evaluate(self):
# run episodes until evaluation is complete
while not self.env.evaluation_complete:
episode_reward = self.play_single_episode()
pprint(self.env.results)
self.env.close()
class StableA2C():
def __init__(self,
env_path,
train,
evaluate,
policy_name='CnnPolicy',
save_dir='./model_files/',
eval_seeds=[]):
self.save_dir = save_dir
if not os.path.exists(save_dir):
os.makedirs(save_dir)
self.model_path = os.path.join(self.save_dir, 'model_stable_a2c')
self.log_dir = './logs/stable_a2c'
self.policy_name = policy_name
self.evaluate = evaluate
if train:
self.env = ObstacleTowerEnv(env_path,
worker_id=0,
retro=True,
realtime_mode=False,
config=train_env_reset_config)
else:
if evaluate:
self.env = ObstacleTowerEnv(env_path,
worker_id=0,
retro=True,
realtime_mode=False,
config=eval_env_reset_config)
self.env = ObstacleTowerEvaluation(self.env, eval_seeds)
else:
self.env = ObstacleTowerEnv(env_path,
worker_id=0,
retro=True,
realtime_mode=True,
config=eval_env_reset_config)
def load_model(self):
print('Loading model from: {}'.format(self.model_path))
model = A2C.load(self.model_path)
model.set_env(self.env)
model.tensorboard_log = self.log_dir
return model
def train(self, timesteps=10000, continue_training=False):
start_time = time.time()
if not continue_training:
print("Initializing from scratch")
model = A2C(self.policy_name,
self.env,
verbose=1,
tensorboard_log=self.log_dir)
else:
model = self.load_model()
print("Restored from {}".format(self.model_path))
model.learn(total_timesteps=timesteps)
print('\nTraining complete. Time taken = {} secs'.format(time.time() -
start_time))
model.save(self.model_path)
def play_single_episode(self):
""" have the trained agent play a single game """
action_space = ActionSpace()
done = False
reward_sum = 0
step_counter = 0
model = self.load_model()
obs = self.env.reset()
try:
print("Playing single episode...")
while not done:
action, _states = model.predict(obs)
obs, reward, done, info = self.env.step(action)
print("{}. Reward: {}, action: {}".format(
step_counter, reward_sum,
action_space.get_full_action_meaning(action)))
self.env.render()
step_counter += 1
reward_sum += reward
except KeyboardInterrupt:
print("Received Keyboard Interrupt. Shutting down.")
finally:
if not self.evaluate:
self.env.close()
print("Environment closed.")
print("Game play completed.")
return reward_sum
def evaluate(self):
""" run episodes until evaluation is complete """
while not self.env.evaluation_complete:
episode_reward = self.play_single_episode()
pprint(self.env.results)
self.env.close()
class WrappedObstacleTowerEnv():
def __init__(self,
environment_filename=None,
docker_training=False,
worker_id=0,
retro=False,
timeout_wait=30,
realtime_mode=False,
num_actions=3,
mobilenet=False,
gray_scale=False,
autoencoder=None,
floor=0):
'''
Arguments:
environment_filename: The file path to the Unity executable. Does not require the extension.
docker_training: Whether this is running within a docker environment and should use a virtual
frame buffer (xvfb).
worker_id: The index of the worker in the case where multiple environments are running. Each
environment reserves port (5005 + worker_id) for communication with the Unity executable.
retro: Resize visual observation to 84x84 (int8) and flattens action space.
timeout_wait: Time for python interface to wait for environment to connect.
realtime_mode: Whether to render the environment window image and run environment at realtime.
'''
self._obstacle_tower_env = ObstacleTowerEnv(environment_filename,
docker_training, worker_id,
retro, timeout_wait,
realtime_mode)
if floor != 0:
self._obstacle_tower_env.floor(floor)
self._flattener = ActionFlattener([3, 3, 2, 3])
self._action_space = self._flattener.action_space
self.mobilenet = mobilenet
self.gray_scale = gray_scale
if mobilenet:
self.image_module = WrappedKerasLayer(retro, self.mobilenet)
self._done = False
if autoencoder:
print("Loading autoencoder from {}".format(autoencoder))
self.autoencoder = build_autoencoder(autoencoder)
print("Done.")
else:
self.autoencoder = None
def action_spec(self):
return self._action_spec
def observation_spec(self):
return self._observation_spec
def gray_process_observation(self, observation):
observation = (observation * 255).astype(np.uint8)
obs_image = Image.fromarray(observation)
obs_image = obs_image.resize((84, 84), Image.NEAREST)
gray_observation = np.mean(np.array(obs_image), axis=-1, keepdims=True)
gray_observation = (gray_observation / 255)
# gray_observation = self.autoencoder.predict(gray_observation)
return gray_observation
def _preprocess_observation(self, observation):
"""
Re-sizes visual observation to 84x84
"""
observation = (observation * 255).astype(np.uint8)
obs_image = Image.fromarray(observation)
obs_image = obs_image.resize((224, 224), Image.NEAREST)
return np.array(obs_image)
def reset(self):
observation = self._obstacle_tower_env.reset()
observation, key, time = observation
self._done = False
if self.mobilenet:
if self.autoencoder:
observation = self.autoencoder.predict(observation[None, :])[0]
return self.image_module(self._preprocess_observation(
observation)), observation, key, time
elif self.gray_scale:
gray_observation = self.gray_process_observation(observation)
if self.autoencoder:
gray_observation = self.autoencoder.predict(
gray_observation[None, :])[0]
return gray_observation, observation
else:
return self._preprocess_observation(observation), observation
def step(self, action):
#if self._done:
# return self.reset()
if action == 0: # forward
action = [1, 0, 0, 0]
elif action == 1: # rotate camera left
action = [0, 1, 0, 0]
elif action == 2: # rotate camera right
action = [0, 2, 0, 0]
elif action == 3: # jump forward
action = [1, 0, 1, 0]
# elif action == 5:
# action = [2, 0, 0, 0]
# elif action == 6:
# action = [0, 0, 0, 1]
# elif action == 7:
# action = [0, 0, 0, 2]
observation, reward, done, info = self._obstacle_tower_env.step(action)
observation, key, time = observation
self._done = done
if self.mobilenet:
if self.autoencoder:
observation = self.autoencoder.predict(observation[None, :])[0]
return (self.image_module(
self._preprocess_observation(observation)), reward, done,
info), observation, key, time
elif self.gray_scale:
gray_observation = self.gray_process_observation(observation)
if self.autoencoder:
gray_observation = self.autoencoder.predict(
gray_observation[None, :])[0]
return (gray_observation, reward, done, info), observation
else:
return (self._preprocess_observation(observation), reward, done,
info), observation
def close(self):
self._obstacle_tower_env.close()
def floor(self, floor):
self._obstacle_tower_env.floor(floor)
class Worker(threading.Thread):
episode_count = 0
running_reward = 0
best_score = 0
global_steps = 0
save_lock = threading.Lock()
def __init__(self, result_queue, params, save_dir):
super(Worker, self).__init__()
self.result_queue = result_queue
self.save_dir = save_dir
self.model_path = os.path.join(self.save_dir, 'model_a3c_distributed')
self.env = ObstacleTowerEnv(params['env_path'],
worker_id=1,
retro=False,
realtime_mode=False,
greyscale=False,
config=train_env_reset_config)
self.action_size = params['action_size']
self._action_lookup = params['action_lookup']
self.input_shape = self.env.observation_space[0].shape # (84, 84, 3)
self._last_health = 99999.
self._last_keys = 0
self.global_model = params['global_model']
self.mirrored_strategy = tf.distribute.MirroredStrategy()
with self.mirrored_strategy.scope():
# self.local_model = CNN(self.action_size, self.input_shape)
self.local_model = CnnGru(self.action_size, self.input_shape)
self.current_time = params['log_timestamp']
train_log_dir = './logs/' + self.current_time + '/worker_1'
self.worker_summary_writer = tf.summary.create_file_writer(
train_log_dir)
self.timesteps = params['timesteps']
self.batch_size = params['batch_size']
self.gamma = params['gamma']
self.lr = params['lr']
self.opt = params['optimizer']
self.eps = np.finfo(np.float32).eps.item()
self.ep_loss = 0.0
def get_updated_reward(self, reward, new_health, new_keys, done):
new_health = float(new_health)
if done: # penalize when game is terminated
self._last_health = 99999.
self._last_keys = 0
reward = -1
else:
# crossing a floor- between [1, 4]
if reward >= 1:
reward += (new_health / 10000)
# found time orb / crossed a floor
if new_health > self._last_health:
reward += 0.1
# found a key
if new_keys > self._last_keys:
reward += 0.1
return reward
def log_worker_metrics(self, episode_reward, avg_reward, loss, step):
with self.worker_summary_writer.as_default():
tf.summary.scalar('reward', episode_reward, step=step)
tf.summary.scalar('moving_reward', avg_reward, step=step)
tf.summary.scalar('loss', loss, step=step)
self.worker_summary_writer.flush()
def run(self):
mem = Memory()
rewards = []
ep_count = 1
timestep = 0
entropy_term = 0
ep_reward = 0.
ep_steps = 0
ep_loss = 0.
done = False
obs = self.env.reset()
state, _, _, _ = obs
while timestep <= self.timesteps:
with tf.GradientTape() as tape:
for i in range(self.batch_size):
# collect experience
# get action as per policy
state = tf.convert_to_tensor(state)
state = tf.expand_dims(state, axis=0)
action_probs, critic_value = self.local_model(
state, training=True)
entropy = -np.sum(action_probs * np.log(action_probs))
entropy_term += entropy
# choose most probable action
action_index = np.random.choice(self.action_size,
p=np.squeeze(action_probs))
action = self._action_lookup[action_index]
# perform action in game env
for i in range(4): # frame skipping
obs, reward, done, _ = self.env.step(action)
state, new_keys, new_health, cur_floor = obs
reward = self.get_updated_reward(
reward, new_health, new_keys, done)
self._last_health = new_health
self._last_keys = new_keys
ep_reward += reward
ep_steps += 1
timestep += 1
# store experience
mem.store(action_prob=action_probs[0, action_index],
value=critic_value[0, 0],
reward=reward)
if done:
break
# backpropagation
total_loss = self.local_model.compute_loss(
mem, state, done, self.gamma, self.eps, entropy_term)
ep_loss += total_loss
Worker.global_steps += ep_steps
grads = tape.gradient(total_loss,
self.local_model.trainable_variables
) # calculate local gradients
# self.opt.apply_gradients(zip(grads, self.global_model.trainable_variables)) # send local gradients to global model
# self.local_model.set_weights(self.global_model.get_weights()) # update local model with new weights
mem.clear()
if done:
rewards.append(ep_reward)
Worker.running_reward = sum(rewards[-10:]) / 10
self.log_worker_metrics(ep_reward, Worker.running_reward,
ep_loss, ep_count)
print(
"Episode: {} | Average Reward: {:.3f} | Episode Reward: {:.3f} | Loss: {:.3f} | Steps: {} | Total Steps: {} | Worker: {}"
.format(Worker.episode_count, Worker.running_reward,
ep_reward, ep_loss, ep_steps, Worker.global_steps,
1))
self.result_queue.put((Worker.running_reward, total_loss))
Worker.episode_count += 1
ep_count += 1
obs = self.env.reset()
state, _, _, _ = obs
# use a lock to save local model and to print to prevent data races.
if ep_reward > Worker.best_score:
with Worker.save_lock:
print(
'\nSaving best model to: {}, episode score: {}\n'.
format(self.model_path, ep_reward))
keras.models.save_model(self.local_model,
self.model_path)
Worker.best_score = ep_reward
entropy_term = 0
ep_reward = 0.
ep_steps = 0
ep_loss = 0.
keras.models.save_model(self.local_model, self.model_path)
self.result_queue.put(None)
self.env.close()
class WrappedObstacleTowerEnv():
def __init__(
self,
environment_filename=None,
docker_training=False,
worker_id=0,
retro=False,
timeout_wait=3000,
realtime_mode=False,
num_actions=3,
stack_size=4,
mobilenet=False,
gray_scale=False,
floor=0,
visual_theme=0
):
'''
Arguments:
environment_filename: The file path to the Unity executable. Does not require the extension.
docker_training: Whether this is running within a docker environment and should use a virtual
frame buffer (xvfb).
worker_id: The index of the worker in the case where multiple environments are running. Each
environment reserves port (5005 + worker_id) for communication with the Unity executable.
retro: Resize visual observation to 84x84 (int8) and flattens action space.
timeout_wait: Time for python interface to wait for environment to connect.
realtime_mode: Whether to render the environment window image and run environment at realtime.
'''
self._obstacle_tower_env = ObstacleTowerEnv(environment_filename,
docker_training,
worker_id,
retro,
timeout_wait,
realtime_mode)
if floor is not 0:
self._obstacle_tower_env.floor(floor)
self.start_floor = floor
self.current_floor = floor
self.mobilenet = mobilenet
self.gray_scale = gray_scale
self.retro = retro
if mobilenet:
self.state_size = [1280]
elif gray_scale:
self.state_size = [84, 84, 1]
elif retro:
self.state_size = [84, 84, 3]
else:
self.state_size = [168, 168, 3]
self.stack_size = stack_size
self.stack = [np.random.random(self.state_size).astype(np.float32) for _ in range(self.stack_size)]
self.total_reward = 0
self.current_reward = 0
self.max_floor = 25
self.visual_theme = visual_theme
self.id = worker_id
def gray_preprocess_observation(self, observation):
'''
Re-sizes obs to 84x84 and compresses to grayscale
'''
observation = (observation * 255).astype(np.uint8)
obs_image = Image.fromarray(observation)
obs_image = obs_image.resize((84, 84), Image.NEAREST)
gray_observation = np.mean(np.array(obs_image),axis=-1,keepdims=True)
return gray_observation / 255
def mobile_preprocess_observation(self, observation):
"""
Re-sizes obs to 224x224 for mobilenet
"""
observation = (observation * 255).astype(np.uint8)
obs_image = Image.fromarray(observation)
obs_image = obs_image.resize((224, 224), Image.NEAREST)
return self.mobilenet(np.array(obs_image))
def reset(self):
# Reset env, stack and floor
# (We save state as an attribute so child objects can access it)
config = {"total-floors": 15}
self.state = self._obstacle_tower_env.reset(config)
self.state, reward, done, info = self._obstacle_tower_env.step(18)
self.current_floor = self.start_floor
self.stack = [np.random.random(self.state_size).astype(np.float32) for _ in range(self.stack_size)]
self.total_reward = 0
self.current_reward = 0
# Preprocess current obs and add to stack
if self.retro:
observation = (self.state / 255).astype(np.float32)
else:
observation, key, time = self.state
if self.mobilenet:
observation = self.mobile_preprocess_observation(observation)
elif self.gray_scale:
observation = self.gray_preprocess_observation(observation)
self.stack = self.stack[1:] + [observation]
# Build our state (MUST BE A TUPLE)
#one_hot_floor = tf.one_hot(self.current_floor, self.max_floor).numpy()
one_hot_floor = np.zeros(self.max_floor)
one_hot_floor[self.current_floor] += 1
floor_data = np.append(one_hot_floor, self.current_reward).astype(np.float32)
stacked_state = np.concatenate(self.stack, axis=-1).astype(np.float32)
if self.retro is True:
ret_state = (stacked_state, floor_data)
else:
# Clip time to 2000, then normalize
time = (2000. if time > 2000 else time) / 2000.
key_time_data = np.array([key, time]).astype(np.float32)
#key_time_data = np.array([key]).astype(np.float32)
ret_state = (stacked_state, floor_data, key_time_data)
return ret_state, info
def step(self, action):
# Convert int action to vector required by the env
if self.retro:
if action == 0: # forward
action = 18
elif action == 1: # rotate camera left
action = 24
elif action == 2: # rotate camera right
action = 30
elif action == 3: # jump forward
action = 21
elif action == 4:
action = 6
elif action == 5:
action = 12
else:
if action == 0: # forward
action = [1, 0, 0, 0]
elif action == 1: # rotate camera left
action = [1, 1, 0, 0]
elif action == 2: # rotate camera right
action = [1, 2, 0, 0]
elif action == 3: # jump forward
action = [1, 0, 1, 0]
# Take the step and record data
# (We save state as an attribute so child objects can access it)
self.state, reward, done, info = self._obstacle_tower_env.step(action)
# Keep track of current floor reward and total reward
if reward >= 0.95:
self.current_floor += 1
self.current_reward = 0
done = True
else:
self.current_reward += reward
self.total_reward += reward
if (done and reward < 0.95) or self.current_floor == 15:
# Save info and reset when an episode ends
info["episode_info"] = {"floor": self.current_floor, "total_reward": self.total_reward}
ret_state, _ = self.reset()
else:
# Preprocess current obs and add to stack
if self.retro:
observation = (self.state / 255).astype(np.float32)
else:
observation, key, time = self.state
if self.mobilenet:
observation = self.mobile_preprocess_observation(observation)
elif self.gray_scale:
observation = self.gray_preprocess_observation(observation)
self.stack = self.stack[1:] + [observation]
# Build our state (MUST BE A TUPLE)
#one_hot_floor = tf.one_hot(self.current_floor, self.max_floor).numpy()
one_hot_floor = np.zeros(self.max_floor)
one_hot_floor[self.current_floor] += 1
floor_data = np.append(one_hot_floor, self.current_reward).astype(np.float32)
stacked_state = np.concatenate(self.stack, axis=-1).astype(np.float32)
if self.retro is True:
ret_state = (stacked_state, floor_data)
else:
# Clip time to 2000, then normalize
time = (2000. if time > 2000 else time) / 2000.
key_time_data = np.array([key, time]).astype(np.float32)
#key_time_data = np.array([key]).astype(np.float32)
ret_state = (stacked_state, floor_data, key_time_data)
return ret_state, reward, done, info
def close(self):
self._obstacle_tower_env.close()
def floor(self, floor):
self._obstacle_tower_env.floor(floor)
self.start_floor = floor