def main():
if len(sys.argv) != 2:
sys.stderr.write('Usage: python record_tail.py <start_floor>\n')
sys.exit(1)
start_floor = int(sys.argv[1])
viewer = EnvInteractor()
env = ObstacleTowerEnv(os.environ['OBS_TOWER_PATH'],
worker_id=random.randrange(11, 20))
while True:
seed = select_seed(floor=start_floor)
env.seed(seed)
env.floor(start_floor)
obs = env.reset()
viewer.reset()
record_episode(seed, env, viewer, obs, max_steps=MAX_STEPS)
def main():
if len(sys.argv) != 2:
sys.stderr.write('Usage: record_improve.py <recording_path>\n')
os.exit(1)
rec = Recording(sys.argv[1])
env = ObstacleTowerEnv(os.environ['OBS_TOWER_PATH'],
worker_id=random.randrange(11, 20))
try:
env.seed(rec.seed)
if rec.floor:
env.floor(rec.floor)
env.reset()
i = 0
for i, (action, rew) in enumerate(zip(rec.actions, rec.rewards)):
_, real_rew, done, _ = env.step(action)
if not np.allclose(real_rew, rew):
print('mismatching result at step %d' % i)
sys.exit(1)
if done != (i == rec.num_steps - 1):
print('invalid done result at step %d' % i)
sys.exit(1)
print('match succeeded')
finally:
env.close()
import os
from obstacle_tower_env import ObstacleTowerEnv
env = ObstacleTowerEnv(os.environ['OBS_TOWER_PATH'], worker_id=0)
env.seed(72)
env.floor(12)
env.reset()
for action in [
18, 18, 18, 18, 18, 18, 30, 24, 24, 21, 18, 18, 30, 18, 18, 18, 18, 18,
18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 24, 18, 30, 18, 18,
18, 18, 18, 18, 18, 18, 18, 18, 30, 30, 30, 30, 24, 24, 6, 6, 6, 6, 6,
6, 6, 6, 30, 30, 30, 30, 30, 18, 24, 24, 24, 6, 6, 6, 6, 6, 6, 24, 18,
24, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
18, 18, 18, 18, 18, 18, 18, 18, 18, 6, 6, 6, 6, 24, 24, 24, 18, 30, 18,
18, 30, 18, 30, 30, 18, 18, 18, 18, 18, 18, 18, 18, 30, 24, 24, 30, 30,
24, 24, 24, 30, 30, 30, 30, 30, 18, 18, 18, 18, 30, 30, 30, 30, 30, 30,
30, 30, 30, 30, 30, 30, 30, 30, 24, 24, 24, 24, 24, 24, 24, 18, 18, 18,
18, 18, 18, 18, 18, 18, 18, 18, 24, 18, 18, 30, 18, 18, 18, 18, 18, 18,
18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
18, 18, 24, 18, 30, 18, 18, 18, 18, 30, 30, 30, 18, 18, 18, 18, 18, 18,
18, 18, 18, 18, 18, 30, 18, 18, 30, 18, 18, 18, 18, 18, 18, 18, 18, 18,
18, 18, 18, 18, 30, 24, 24, 24, 24, 24, 24, 24, 24, 18, 30, 18, 18, 18,
18, 30, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 30,
30, 30, 30, 30, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
18, 18, 30, 24, 21, 18, 24, 24, 24, 24, 18, 18, 18, 24, 18, 18, 18, 18,
30, 18, 18, 24, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
24, 24, 24, 24, 24, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
18, 18, 30, 30, 30, 18, 18, 30, 30, 30, 30, 30, 30, 12, 12, 30, 30, 30,
30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 18, 18, 18, 18, 18, 18, 18, 18,
class Worker(object):
def __init__(self,
envpath,
wid,
retro,
realtime_mode,
env_seed=0,
env_floor=0):
self.wid = wid
self.env = ObstacleTowerEnv(environment_filename=envpath,
worker_id=wid,
retro=retro,
realtime_mode=realtime_mode)
self.kprun = GLOBAL_KPRUN
self.tableAction = self.createActionTable()
# 設定關卡
self.env_seed = env_seed
self.env_floor = env_floor
self.step = 0
self.summary = tf.Summary(value=[
tf.Summary.Value(tag="Stage_reward " + str(self.wid),
simple_value=0)
])
self.kprun.train_writer.add_summary(self.summary, 0)
def createActionTable(self):
tableAction = []
for a in range(0, 3):
for b in range(0, 3):
for c in range(0, 2):
tableAction.append([a, b, c, 0])
# print("Action option: ", tableAction[0:17])
return tableAction
def reward_compute(self, done, reward_total, keys, previous_keys, reward,
previous_reward, time_remaining,
previous_time_remaining, previous_stage_time_remaining):
# 定義獎勵公式
# reward 是從環境傳來的破關數
# keys 是撿到鑰匙的數量
# time_remaining 是剩餘時間
# 過關最大獎勵為10
# 一把鑰匙為5
# 時間果實暫時只給0.5，因為結束會結算剩餘時間，會有獎勵累加的問題。
# 如果過關，給予十倍過關獎勵 - (場景開始的時間-剩餘時間)/1000
# print("time_remaining ", time_remaining,
# " previous_time_remaining ", previous_time_remaining,
# " reward ", reward)
# 通過一個會開門的綠門會加0.1
if (reward - previous_reward) > 0 and (reward - previous_reward) < 0.3:
reward_total += 3
elif (reward - previous_reward) > 0.9:
# ***如果剩餘時間比場景時間多會變成加分獎勵，可能會極大增加Agent吃時間果實的機率。
# ***另一種方式是剩餘的時間直接/1000加上去，這樣就沒有累加效果。
print("Pass ", reward, " Stage!")
# reward_total += (reward - previous_reward) * 100 - \
# (previous_stage_time_remaining - time_remaining)
reward_total += 200
# 過關之後把時間留到下一關，儲存這回合時間供下次計算過關使用
previous_time_remaining = time_remaining
previous_stage_time_remaining = time_remaining
# Lesson 1 repeat
if reward > 6.5:
# self.total_step +=1
# if self.total_step >=5:
# done = True
# return reward_total, previous_stage_time_remaining, done
self.env.seed(np.random.randint(5))
# env.reset()
done = True
return reward_total, previous_stage_time_remaining, done
# 假設過關的時候有順便吃到果實或鑰匙，所以預設為同時可以加成
if previous_keys > keys:
# print("Get Key")
reward_total += 5
if previous_time_remaining < time_remaining and previous_time_remaining != 0:
# print("Get time power up")
reward_total += 2
else:
reward_total -= 0.5
if done and previous_time_remaining > 100:
print("Agent died")
# 如果剩餘時間越多就掛點，扣更多
# reward_total -= (10 + time_remaining / 100)
reward_total -= 100
return reward_total, previous_stage_time_remaining, done
def work(self):
global GLOBAL_EP, GLOBAL_RUNNING_R, GLOBAL_UPDATE_COUNTER
# 設定關卡
self.env.seed(self.env_seed)
self.env.floor(self.env_floor)
# 只要還沒達到目標回合就LOOP
while not COORD.should_stop():
# 紀錄步數
self.step += 1
# 重設關卡
obs = self.env.reset()
# 初始化
done = False
stage_reward = 0.0
reward = 0
keys = 0
# 檢查是否有吃到加時間的，如果是第一回合出來沒有time_remaining，事先定義
time_remaining = 3000
previous_stage_time_remaining = time_remaining
# 預處理圖像
# previous_preprocessed_observation_image = np.reshape(obs[0], [-1])
previous_preprocessed_observation_image = obs[0]
buffer_s, buffer_a, buffer_r = [], [], []
# 只要沒死
while not done:
# 如果模型正在更新就等待更新完成
if not ROLLING_EVENT.is_set():
# 等待更新完成
ROLLING_EVENT.wait()
# 清除記憶體，使用新的代理收集資料
buffer_s, buffer_a, buffer_r = [], [], []
# 儲存上一個動作狀態，供計算獎勵用
previous_keys = keys
previous_reward = reward
previous_time_remaining = time_remaining
# 根據上一次的狀態決定動作
action = self.kprun.choose_action(
previous_preprocessed_observation_image)
action = np.clip(np.random.normal(action, 1.), *[6, 12])
# 做出動作，獲得場景資訊,已過關數,代理資訊
observation, reward, done, info = self.env.step(
np.array(self.tableAction[int(action)]))
# 預處理模型需要的資料
observation_image, keys, time_remaining = observation
# preprocessed_observation_image = np.reshape(
# observation_image, [-1])
preprocessed_observation_image = observation_image
stage_reward, previous_stage_time_remaining, done = self.reward_compute(
done=done,
reward_total=stage_reward,
keys=keys,
previous_keys=previous_keys,
reward=reward,
previous_reward=previous_reward,
time_remaining=time_remaining,
previous_time_remaining=previous_time_remaining,
previous_stage_time_remaining=previous_stage_time_remaining
)
# Normalize reward~不知道中文怎麼打
stage_reward = stage_reward + 8 / 8
# 把這次狀態存入 記憶體
buffer_s.append(np.array([preprocessed_observation_image]))
buffer_a.append(action)
buffer_r.append(stage_reward)
# 儲存下一步要參考的圖像
previous_preprocessed_observation_image = preprocessed_observation_image
# 達到更新時，自己先做處理。
GLOBAL_UPDATE_COUNTER += 1
# 太多自己就先處理更新
if len(buffer_s) == EP_LEN - \
1 or GLOBAL_UPDATE_COUNTER >= MIN_BATCH_SIZE:
v_s_ = self.kprun.get_v(preprocessed_observation_image)
# 計算折扣獎勵
discounted_r = []
for r in buffer_r[::-1]:
v_s_ = r + GAMMA * v_s_
discounted_r.append(v_s_)
discounted_r.reverse()
# 整理維度
bs, ba, br = np.vstack(buffer_s), np.vstack(
buffer_a), np.array(discounted_r)[:, np.newaxis]
# 把資料放入共享記憶體
QUEUE.put(bs)
QUEUE.put(ba)
QUEUE.put(br)
# print("len(buffer_s)", len(buffer_s))
# print("bs.shape", bs.shape)
# 清空暫存
buffer_s, buffer_a, buffer_r = [], [], []
# 如果整個模型步數到達最小BATCH 就整個更新
if GLOBAL_UPDATE_COUNTER >= MIN_BATCH_SIZE:
# 停止收集資料
ROLLING_EVENT.clear()
# 更新PPO
UPDATE_EVENT.set()
# 達到最多EP停止訓練
if GLOBAL_EP >= EP_MAX:
COORD.request_stop()
break
# 紀錄獎勵
self.summary = tf.Summary(value=[
tf.Summary.Value(tag="Stage_reward " + str(self.wid),
simple_value=stage_reward)
])
self.kprun.train_writer.add_summary(self.summary, self.step)
GLOBAL_EP += 1
print(
'{0:.1f}%'.format(GLOBAL_EP / EP_MAX * 100),
'|W%i' % self.wid,
'|Ep_r: %.2f' % stage_reward,
)
self.env.close()
def main():
basicConfig(level=INFO)
env = ObstacleTowerEnv(str(PRJ_ROOT / 'obstacletower'), retro=False, worker_id=9)
done = False
env.floor(1)
env.reset()
screen = Screen()
random_actor = RandomRepeatActor(continue_rate=0.9)
random_actor.reset(schedules=[
(Action.CAMERA_RIGHT, 3),
(Action.CAMERA_LEFT, 6),
(Action.CAMERA_RIGHT, 3),
(Action.NOP, 5),
(Action.FORWARD, 8),
(Action.RIGHT, 2),
(Action.LEFT, 4),
(Action.RIGHT, 2),
])
frame_history = FrameHistory(env)
moving_checker = MovingChecker(frame_history)
position_estimator = PositionEstimator(moving_checker)
map_observation = MapObservation(position_estimator, moving_checker)
event_handlers: List[EventHandler] = [
frame_history,
moving_checker,
position_estimator,
map_observation,
]
while not done:
for h in event_handlers:
h.begin_loop()
screen.show("original", frame_history.last_frame)
cv2.waitKey(0)
for h in event_handlers:
h.before_step()
action = random_actor.decide_action(moving_checker.did_move)
obs, reward, done, info = env.step(action)
if reward != 0:
logger.info(f"Get Reward={reward} Keys={obs[1]}")
# logger.info(f"Keys={obs[1]} Time Remain={obs[2]}")
params = EventParamsAfterStep(action, obs, reward, done, info)
for h in event_handlers:
h.after_step(params)
screen.show("map", map_observation.concat_images())
if len(frame_history.small_frame_pixel_diffs) > 0:
f1 = frame_history.small_frame_pixel_diffs[-1]
if len(frame_history.small_frame_pixel_diffs) > 1:
f2 = frame_history.small_frame_pixel_diffs[-2]
f1 = np.concatenate((f2, f1), axis=1)
screen.show("diff", f1)
for h in event_handlers:
h.end_loop()
class WrappedObstacleTowerEnv():
def __init__(self,
environment_filename=None,
docker_training=False,
worker_id=0,
retro=False,
timeout_wait=30,
realtime_mode=False,
num_actions=3,
mobilenet=False,
gray_scale=False,
autoencoder=None,
floor=0):
'''
Arguments:
environment_filename: The file path to the Unity executable. Does not require the extension.
docker_training: Whether this is running within a docker environment and should use a virtual
frame buffer (xvfb).
worker_id: The index of the worker in the case where multiple environments are running. Each
environment reserves port (5005 + worker_id) for communication with the Unity executable.
retro: Resize visual observation to 84x84 (int8) and flattens action space.
timeout_wait: Time for python interface to wait for environment to connect.
realtime_mode: Whether to render the environment window image and run environment at realtime.
'''
self._obstacle_tower_env = ObstacleTowerEnv(environment_filename,
docker_training, worker_id,
retro, timeout_wait,
realtime_mode)
if floor != 0:
self._obstacle_tower_env.floor(floor)
self._flattener = ActionFlattener([3, 3, 2, 3])
self._action_space = self._flattener.action_space
self.mobilenet = mobilenet
self.gray_scale = gray_scale
if mobilenet:
self.image_module = WrappedKerasLayer(retro, self.mobilenet)
self._done = False
if autoencoder:
print("Loading autoencoder from {}".format(autoencoder))
self.autoencoder = build_autoencoder(autoencoder)
print("Done.")
else:
self.autoencoder = None
def action_spec(self):
return self._action_spec
def observation_spec(self):
return self._observation_spec
def gray_process_observation(self, observation):
observation = (observation * 255).astype(np.uint8)
obs_image = Image.fromarray(observation)
obs_image = obs_image.resize((84, 84), Image.NEAREST)
gray_observation = np.mean(np.array(obs_image), axis=-1, keepdims=True)
gray_observation = (gray_observation / 255)
# gray_observation = self.autoencoder.predict(gray_observation)
return gray_observation
def _preprocess_observation(self, observation):
"""
Re-sizes visual observation to 84x84
"""
observation = (observation * 255).astype(np.uint8)
obs_image = Image.fromarray(observation)
obs_image = obs_image.resize((224, 224), Image.NEAREST)
return np.array(obs_image)
def reset(self):
observation = self._obstacle_tower_env.reset()
observation, key, time = observation
self._done = False
if self.mobilenet:
if self.autoencoder:
observation = self.autoencoder.predict(observation[None, :])[0]
return self.image_module(self._preprocess_observation(
observation)), observation, key, time
elif self.gray_scale:
gray_observation = self.gray_process_observation(observation)
if self.autoencoder:
gray_observation = self.autoencoder.predict(
gray_observation[None, :])[0]
return gray_observation, observation
else:
return self._preprocess_observation(observation), observation
def step(self, action):
#if self._done:
# return self.reset()
if action == 0: # forward
action = [1, 0, 0, 0]
elif action == 1: # rotate camera left
action = [0, 1, 0, 0]
elif action == 2: # rotate camera right
action = [0, 2, 0, 0]
elif action == 3: # jump forward
action = [1, 0, 1, 0]
# elif action == 5:
# action = [2, 0, 0, 0]
# elif action == 6:
# action = [0, 0, 0, 1]
# elif action == 7:
# action = [0, 0, 0, 2]
observation, reward, done, info = self._obstacle_tower_env.step(action)
observation, key, time = observation
self._done = done
if self.mobilenet:
if self.autoencoder:
observation = self.autoencoder.predict(observation[None, :])[0]
return (self.image_module(
self._preprocess_observation(observation)), reward, done,
info), observation, key, time
elif self.gray_scale:
gray_observation = self.gray_process_observation(observation)
if self.autoencoder:
gray_observation = self.autoencoder.predict(
gray_observation[None, :])[0]
return (gray_observation, reward, done, info), observation
else:
return (self._preprocess_observation(observation), reward, done,
info), observation
def close(self):
self._obstacle_tower_env.close()
def floor(self, floor):
self._obstacle_tower_env.floor(floor)
#!/usr/bin/env python3
from obstacle_tower_env import ObstacleTowerEnv
from matplotlib import pyplot as plt
ENV_PATH = './obstacle-tower-challenge/ObstacleTower/obstacletower'
env = ObstacleTowerEnv(ENV_PATH, retro=False, realtime_mode=True)
# Seeds can be chosen from range of 0-100.
env.seed(5)
# Floors can be chosen from range of 0-24.
env.floor(15)
# The environment provided has a MultiDiscrete action space, where the 4 dimensions are:
# 0. Movement (No-Op/Forward/Back)
# 1. Camera Rotation (No-Op/Counter-Clockwise/Clockwise)
# 2. Jump (No-Op/Jump)
# 3. Movement (No-Op/Right/Left)
print('action space', env.action_space)
# The observation space provided includes a 168x168 image (the camera from the simulation)
# as well as the number of keys held by the agent (0-5) and the amount of time remaining.
print('observation space', env.observation_space)
# Interacting with the environment
obs = env.reset()
class WrappedObstacleTowerEnv():
def __init__(
self,
environment_filename=None,
docker_training=False,
worker_id=0,
retro=False,
timeout_wait=3000,
realtime_mode=False,
num_actions=3,
stack_size=4,
mobilenet=False,
gray_scale=False,
floor=0,
visual_theme=0
):
'''
Arguments:
environment_filename: The file path to the Unity executable. Does not require the extension.
docker_training: Whether this is running within a docker environment and should use a virtual
frame buffer (xvfb).
worker_id: The index of the worker in the case where multiple environments are running. Each
environment reserves port (5005 + worker_id) for communication with the Unity executable.
retro: Resize visual observation to 84x84 (int8) and flattens action space.
timeout_wait: Time for python interface to wait for environment to connect.
realtime_mode: Whether to render the environment window image and run environment at realtime.
'''
self._obstacle_tower_env = ObstacleTowerEnv(environment_filename,
docker_training,
worker_id,
retro,
timeout_wait,
realtime_mode)
if floor is not 0:
self._obstacle_tower_env.floor(floor)
self.start_floor = floor
self.current_floor = floor
self.mobilenet = mobilenet
self.gray_scale = gray_scale
self.retro = retro
if mobilenet:
self.state_size = [1280]
elif gray_scale:
self.state_size = [84, 84, 1]
elif retro:
self.state_size = [84, 84, 3]
else:
self.state_size = [168, 168, 3]
self.stack_size = stack_size
self.stack = [np.random.random(self.state_size).astype(np.float32) for _ in range(self.stack_size)]
self.total_reward = 0
self.current_reward = 0
self.max_floor = 25
self.visual_theme = visual_theme
self.id = worker_id
def gray_preprocess_observation(self, observation):
'''
Re-sizes obs to 84x84 and compresses to grayscale
'''
observation = (observation * 255).astype(np.uint8)
obs_image = Image.fromarray(observation)
obs_image = obs_image.resize((84, 84), Image.NEAREST)
gray_observation = np.mean(np.array(obs_image),axis=-1,keepdims=True)
return gray_observation / 255
def mobile_preprocess_observation(self, observation):
"""
Re-sizes obs to 224x224 for mobilenet
"""
observation = (observation * 255).astype(np.uint8)
obs_image = Image.fromarray(observation)
obs_image = obs_image.resize((224, 224), Image.NEAREST)
return self.mobilenet(np.array(obs_image))
def reset(self):
# Reset env, stack and floor
# (We save state as an attribute so child objects can access it)
config = {"total-floors": 15}
self.state = self._obstacle_tower_env.reset(config)
self.state, reward, done, info = self._obstacle_tower_env.step(18)
self.current_floor = self.start_floor
self.stack = [np.random.random(self.state_size).astype(np.float32) for _ in range(self.stack_size)]
self.total_reward = 0
self.current_reward = 0
# Preprocess current obs and add to stack
if self.retro:
observation = (self.state / 255).astype(np.float32)
else:
observation, key, time = self.state
if self.mobilenet:
observation = self.mobile_preprocess_observation(observation)
elif self.gray_scale:
observation = self.gray_preprocess_observation(observation)
self.stack = self.stack[1:] + [observation]
# Build our state (MUST BE A TUPLE)
#one_hot_floor = tf.one_hot(self.current_floor, self.max_floor).numpy()
one_hot_floor = np.zeros(self.max_floor)
one_hot_floor[self.current_floor] += 1
floor_data = np.append(one_hot_floor, self.current_reward).astype(np.float32)
stacked_state = np.concatenate(self.stack, axis=-1).astype(np.float32)
if self.retro is True:
ret_state = (stacked_state, floor_data)
else:
# Clip time to 2000, then normalize
time = (2000. if time > 2000 else time) / 2000.
key_time_data = np.array([key, time]).astype(np.float32)
#key_time_data = np.array([key]).astype(np.float32)
ret_state = (stacked_state, floor_data, key_time_data)
return ret_state, info
def step(self, action):
# Convert int action to vector required by the env
if self.retro:
if action == 0: # forward
action = 18
elif action == 1: # rotate camera left
action = 24
elif action == 2: # rotate camera right
action = 30
elif action == 3: # jump forward
action = 21
elif action == 4:
action = 6
elif action == 5:
action = 12
else:
if action == 0: # forward
action = [1, 0, 0, 0]
elif action == 1: # rotate camera left
action = [1, 1, 0, 0]
elif action == 2: # rotate camera right
action = [1, 2, 0, 0]
elif action == 3: # jump forward
action = [1, 0, 1, 0]
# Take the step and record data
# (We save state as an attribute so child objects can access it)
self.state, reward, done, info = self._obstacle_tower_env.step(action)
# Keep track of current floor reward and total reward
if reward >= 0.95:
self.current_floor += 1
self.current_reward = 0
done = True
else:
self.current_reward += reward
self.total_reward += reward
if (done and reward < 0.95) or self.current_floor == 15:
# Save info and reset when an episode ends
info["episode_info"] = {"floor": self.current_floor, "total_reward": self.total_reward}
ret_state, _ = self.reset()
else:
# Preprocess current obs and add to stack
if self.retro:
observation = (self.state / 255).astype(np.float32)
else:
observation, key, time = self.state
if self.mobilenet:
observation = self.mobile_preprocess_observation(observation)
elif self.gray_scale:
observation = self.gray_preprocess_observation(observation)
self.stack = self.stack[1:] + [observation]
# Build our state (MUST BE A TUPLE)
#one_hot_floor = tf.one_hot(self.current_floor, self.max_floor).numpy()
one_hot_floor = np.zeros(self.max_floor)
one_hot_floor[self.current_floor] += 1
floor_data = np.append(one_hot_floor, self.current_reward).astype(np.float32)
stacked_state = np.concatenate(self.stack, axis=-1).astype(np.float32)
if self.retro is True:
ret_state = (stacked_state, floor_data)
else:
# Clip time to 2000, then normalize
time = (2000. if time > 2000 else time) / 2000.
key_time_data = np.array([key, time]).astype(np.float32)
#key_time_data = np.array([key]).astype(np.float32)
ret_state = (stacked_state, floor_data, key_time_data)
return ret_state, reward, done, info
def close(self):
self._obstacle_tower_env.close()
def floor(self, floor):
self._obstacle_tower_env.floor(floor)
self.start_floor = floor