class DoomEnv(gym.Env, EzPickle):
metadata = {
'render.modes': ['human', 'rgb_array'],
'video.frames_per_second': 35
}
def __init__(self, level='deathmatch', obs_type='ram'):
# super(DoomEnv, self).__init__()
EzPickle.__init__(self, level.split('.')[0], obs_type)
assert obs_type in ('ram', 'image')
level = level.split('.')[0]
Config.init(level)
self.curr_seed = 0
self.game = DoomGame()
self.lock = (DoomLock()).get_lock()
self.level = level
self.obs_type = obs_type
self.tick = 4
self._mode = 'algo'
self.is_render_in_human_mode = True
self.is_game_initialized = False
self.is_level_loaded = False
self.viewer = None
self.set_game(self.level, resolution=None, render=True)
print()
# todo: add frame skip option by using tick
def step(self, action):
reward = 0.0
# self.tick = 4
if self._mode == 'algo':
if self.tick:
reward = self.game.make_action(action, self.tick)
else:
reward = self.game.make_action(action)
# self.game.set_action(action)
# self.game.advance_action(4)
# reward = self.game.get_last_reward()
return self.get_obs(), reward, self.isDone(), self.get_info()
def reset(self):
if not self.is_game_initialized:
self.__load_level()
self.__init_game()
self.__start_episode()
return self.get_obs()
def render(self, mode='human', **kwargs):
if 'close' in kwargs and kwargs['close']:
if self.viewer is not None:
self.viewer.close()
self.viewer = None
return
if mode == 'human' and not self.is_render_in_human_mode:
return
img = self.get_image()
if mode == 'rgb_array':
return img
elif mode is 'human':
if self.viewer is None:
self.viewer = rendering.SimpleImageViewer()
self.viewer.imshow(img)
def close(self):
with self.lock:
self.game.close()
def seed(self, seed=None):
self.curr_seed = seeding.hash_seed(seed) % 2**32
return [self.curr_seed]
# ================================== GETTERS SETTERS ===============================================================
def set_game(self, level, resolution, render):
self.__configure()
self.__load_level(level)
self.__set_resolution(resolution)
self.__set_obs_and_ac_space()
self.__set_player(render)
def __configure(self, lock=None, **kwargs):
self.seed()
if lock is not None:
self.lock = lock
def __load_level(self, level=None):
if level is not None:
self.level = level.split('.')[0]
self.is_level_loaded = False
if self.is_level_loaded:
return
if self.is_game_initialized:
self.is_game_initialized = False
self.game.close()
self.game = DoomGame()
if not self.is_game_initialized:
self.game.set_vizdoom_path(Config.VIZDOOM_PATH)
self.game.set_doom_game_path(Config.FREEDOOM_PATH)
# Common settings
self.record_file_path = Config.RECORD_FILE_PATH
self.game.load_config(Config.VIZDOOM_SCENARIO_PATH +
Config.DOOM_SETTINGS[self.level][Config.CONFIG])
self.game.set_doom_scenario_path(
Config.VIZDOOM_SCENARIO_PATH +
Config.DOOM_SETTINGS[self.level][Config.SCENARIO])
if Config.DOOM_SETTINGS[self.level][Config.MAP] != '':
self.game.set_doom_map(
Config.DOOM_SETTINGS[self.level][Config.MAP])
self.game.set_doom_skill(
Config.DOOM_SETTINGS[self.level][Config.DIFFICULTY])
self.allowed_actions = Config.DOOM_SETTINGS[self.level][Config.ACTIONS]
self.available_game_variables = Config.DOOM_SETTINGS[self.level][
Config.GAME_VARIABLES]
self.is_level_loaded = True
def __set_resolution(self, resolution=None):
if resolution is None:
resolution = Config.DEFAULT_SCREEN_RESOLUTION
resolution_l = resolution.lower()
if resolution_l not in resolutions:
raise gym.error.Error(
'Error - The specified resolution "{}" is not supported by Vizdoom.\n The list of valid'
'resolutions: {}'.format(resolution, resolutions))
if '_' in resolution_l:
resolution_l = resolution_l.split('_')[1]
self.scr_width = int(resolution_l.split("x")[0])
self.scr_height = int(resolution_l.split("x")[1])
self.game.set_screen_resolution(
getattr(ScreenResolution,
'RES_{}X{}'.format(self.scr_width, self.scr_height)))
self.screen_format = self.game.get_screen_format()
self.screen_height = self.game.get_screen_height()
self.screen_width = self.game.get_screen_width()
def __set_obs_and_ac_space(self):
if self.obs_type == 'ram':
self.observation_space = spaces.Box(
low=0,
high=255,
dtype=np.uint8,
shape=(len(self.available_game_variables), ))
elif self.obs_type == 'image':
# self.observation_space = self.screen_resized
self.observation_space = spaces.Box(low=0,
high=255,
shape=(self.scr_height,
self.scr_width, 3),
dtype=np.uint8)
else:
raise error.Error('Unrecognized observation type: {}'.format(
self.obs_type))
if self.screen_format in inverted_screen_formats:
self.dummy_screen = np.zeros(shape=(3, self.scr_height,
self.scr_width),
dtype=np.uint8)
else:
self.dummy_screen = np.zeros(shape=(self.scr_height,
self.scr_width, 3),
dtype=np.uint8)
self.dummy_ram = [0] * len(self.available_game_variables)
self.available_action_codes = [
list(a)
for a in it.product([0, 1],
repeat=self.game.get_available_buttons_size())
]
# self.__delete_conflict_actions()
self.action_space = spaces.MultiDiscrete(
[len(self.available_action_codes)])
def __set_player(self, render=True):
self.game.set_window_visible(render)
self.game.set_mode(Mode.PLAYER)
def __init_game(self):
try:
with self.lock:
self.game.init()
self.is_game_initialized = True
except (ViZDoomUnexpectedExitException, ViZDoomErrorException):
raise error.Error('Could not start the game.')
def __start_episode(self):
if self.curr_seed > 0:
self.game.set_seed(self.curr_seed)
self.curr_seed = 0
if self.record_file_path:
self.game.new_episode(self.record_file_path)
else:
self.game.new_episode()
return
def getState(self):
return self.game.get_state()
def getLastAction(self):
return self.game.get_last_action()
def getButtonsNames(self, action):
return action_to_buttons(self.allowed_actions, action)
def get_info(self):
info = {
"LEVEL": self.level,
"TOTAL_REWARD": round(self.game.get_total_reward(), 4)
}
state_variables = self.get_ram()
for i in range(len(self.available_game_variables)):
info[self.available_game_variables[i]] = state_variables[i]
return info
def get_ram(self):
if not self.is_game_initialized:
raise NotImplementedError(
"The game was not initialized. Run env.reset() first!")
try:
ram = self.getState().game_variables
except AttributeError:
ram = self.dummy_ram
return ram
def get_image(self):
try:
screen = self.getState().screen_buffer.copy()
except AttributeError:
screen = self.dummy_screen
return self.invert_screen(screen)
def get_obs(self):
if self.obs_type == 'ram':
return self.get_ram()
elif self.obs_type == 'image':
return self.get_image()
def isDone(self):
return self.game.is_episode_finished() or self.game.is_player_dead(
) or self.getState() is None
# =========================================== ==============================================================
def invert_screen(self, img):
if self.screen_format in inverted_screen_formats:
return np.rollaxis(img, 0, 3)
else:
return img
def __delete_conflict_actions(self):
if self._mode == 'human':
return
action_codes_copy = self.available_action_codes.copy()
print("Initial actions size: " + str(len(action_codes_copy)))
for i in tqdm.trange(len(self.available_action_codes)):
action = self.available_action_codes[i]
ac_names = action_to_buttons(self.allowed_actions, action)
if all(elem in ac_names
for elem in ['MOVE_LEFT', 'MOVE_RIGHT']) or all(
elem in ac_names
for elem in ['MOVE_BACKWARD', 'MOVE_FORWARD']) or all(
elem in ac_names
for elem in ['TURN_RIGHT', 'TURN_LEFT']) or all(
elem in ac_names for elem in
['SELECT_NEXT_WEAPON', 'SELECT_PREV_WEAPON']):
action_codes_copy.remove(action)
print("Final actions size: " + str(len(action_codes_copy)))
self.available_action_codes = action_codes_copy
def __initHumanPlayer(self):
self._mode = 'human'
self.__load_level()
self.game.add_game_args('+freelook 1')
self.game.set_window_visible(True)
self.game.set_mode(Mode.SPECTATOR)
self.is_render_in_human_mode = False
self.__init_game()
def advanceAction(self, tick=0):
try:
if tick:
self.game.advance_action(tick)
else:
self.game.advance_action()
return True
except ViZDoomUnexpectedExitException:
return False
def playHuman(self):
self.__initHumanPlayer()
while not self.game.is_episode_finished(
) and not self.game.is_player_dead():
self.advanceAction()
state = self.getState()
if state is None:
if self.record_file_path is None:
self.game.new_episode()
else:
self.game.new_episode(self.record_file_path)
state = self.getState()
total_reward = self.game.get_total_reward()
info = self.get_info()
info["TOTAL_REWARD"] = round(total_reward, 4)
print('===============================')
print('State: #' + str(state.number))
print('Action: \t' + str(self.game.get_last_action()) +
'\t (=> only allowed actions)')
print('Reward: \t' + str(self.game.get_last_reward()))
print('Total Reward: \t' + str(total_reward))
print('Variables: \n' + str(info))
sleep(0.02857) # 35 fps = 0.02857 sleep between frames
print('===============================')
print('Done')
return
class VizDoomGym(gym.Env):
"""
Wraps a VizDoom environment
"""
def __init__(self):
raise NotImplementedError
def _init(self, mission_file: str, scaled_resolution: list):
"""
:param mission_file: name of the mission (.cfg) to run,
:param scaled_resolution: resolution (height, width) of the video frames
to run training on
"""
super(VizDoomGym, self).__init__()
self.mission_file = mission_file
self._logger = logging.getLogger(__name__)
self._logger.info("Creating environment: VizDoom (%s)",
self.mission_file)
self.deathmatch = True
# distance we need the agent to travel per time-step, otherwise we penalise
self.distance_threshold = 15
self.prev_properties = None
self.properties = None
self.cum_kills = np.array([0])
# Create an instace on VizDoom game, initalise it from a scenario config file
self.env = DoomGame()
self.env.load_config(self.mission_file)
self.env.set_window_visible(False)
self.env.set_screen_format(ScreenFormat.RGB24)
if self.deathmatch:
self.env.add_game_args("-deathmatch")
self.env.set_doom_skill(4)
self._action_frame_repeat = 4
self.env.init()
# Perform config validation:
# Only RGB format with a seperate channel per colour is supported
assert self.env.get_screen_format() == ScreenFormat.RGB24
# Only discrete actions are supported (no delta actions)
self.available_actions = self.env.get_available_buttons()
not_supported_actions = [
Button.LOOK_UP_DOWN_DELTA, Button.TURN_LEFT_RIGHT_DELTA,
Button.MOVE_LEFT_RIGHT_DELTA, Button.MOVE_UP_DOWN_DELTA,
Button.MOVE_FORWARD_BACKWARD_DELTA
]
# print(available_actions)
assert len((set(self.available_actions) - set(not_supported_actions))) \
== len(self.available_actions)
self.metadata['render_modes'] = ['rgb_array']
# Allow only one button to be pressed at a given step
self.action_space = gym.spaces.Discrete(
self.env.get_available_buttons_size() - 1)
self.rows = scaled_resolution[0]
self.columns = scaled_resolution[1]
self.observation_space = gym.spaces.Box(low=0.0,
high=1.0,
shape=(self.rows, self.columns,
3),
dtype=np.float32)
self._rgb_array = None
self.steps = 0
self.global_steps = 0
self.reset()
def _process_image(self, img):
# PIL resize has indexing opposite to numpy array
img = np.array(Image.fromarray(img).resize((self.columns, self.rows)))
img = img.astype(np.float32)
img = img / 255.0
return img
def update_game_variables(self):
"""
Check and update game variables.
"""
# read game variables
new_v = {
k: self.env.get_game_variable(v)
for k, v in game_variables.items()
}
assert all(v.is_integer() or k[-2:] in ['_x', '_y', '_z']
for k, v in new_v.items())
new_v = {
k: (int(v) if v.is_integer() else float(v))
for k, v in new_v.items()
}
health = new_v['health']
armor = new_v['armor']
# check game variables
assert 0 <= health <= 200 or health < 0 and self.env.is_player_dead()
assert 0 <= armor <= 200, (health, armor)
# update actor properties
self.prev_properties = self.properties
self.properties = new_v
def update_reward(self):
"""
Update reward.
"""
# we need to know the current and previous properties
assert self.prev_properties is not None and self.properties is not None
reward = 0
# kill
d = self.properties['score'] - self.prev_properties['score']
if d > 0:
self.cum_kills += d
reward += d * default_reward_values['KILL']
# death
if self.env.is_player_dead():
reward += default_reward_values['DEATH']
# suicide
if self.properties['frag_count'] < self.prev_properties['frag_count']:
reward += default_reward_values['SUICIDE']
# found / lost health
d = self.properties['health'] - self.prev_properties['health']
if d != 0:
if d > 0:
reward += default_reward_values['MEDIKIT']
else:
reward += default_reward_values['INJURED']
# found / lost armor
d = self.properties['armor'] - self.prev_properties['armor']
if d != 0:
if d > 0:
reward += default_reward_values['ARMOR']
# found / lost ammo
d = self.properties['sel_ammo'] - self.prev_properties['sel_ammo']
if d != 0:
if d > 0:
reward += default_reward_values['AMMO']
else:
reward += default_reward_values['USE_AMMO']
# distance
# turn_left = (Button.TURN_LEFT == self.available_actions[action])
# turn_right = (Button.TURN_RIGHT == self.available_actions[action])
# if not (turn_left or turn_right):
diff_x = self.properties['position_x'] - self.prev_properties[
'position_x']
diff_y = self.properties['position_y'] - self.prev_properties[
'position_y']
distance = np.sqrt(diff_x**2 + diff_y**2)
if distance > self.distance_threshold:
reward += default_reward_values['DISTANCE'] * distance
else:
reward += default_reward_values['STANDSTILL']
# living
reward += default_reward_values['LIVING']
return reward
# def increase_difficulty(self):
# self.curr_skill += 1
# self.env.close()
# self.env.set_doom_skill(self.curr_skill)
# self.env.init()
# print('changing skill to', self.curr_skill)
# def update_map(self):
# self.map_level += 1
# map_str = 'map0' + str(self.map_level)
# # go with initial wad file if there's still maps on it
# self.env.close()
# self.env.set_doom_map(map_str)
# self.env.init()
def sub_reset(self):
"""Reset environment"""
self.steps = 0
self.cum_kills = np.array([0])
self.prev_properties = None
self.properties = None
self.env.new_episode()
self._rgb_array = self.env.get_state().screen_buffer
observation = self._process_image(self._rgb_array)
return observation
def reset(self):
observation = self.sub_reset()
return observation
def sub_step(self, action):
"""Take step"""
one_hot_action = np.zeros(self.action_space.n, dtype=int)
one_hot_action[action] = 1
# ALWAYS SPRINTING
one_hot_action = np.append(one_hot_action, [1])
assert len(one_hot_action) == len(self.env.get_available_buttons())
_ = self.env.make_action(list(one_hot_action),
self._action_frame_repeat)
self.update_game_variables()
if self.steps > 1:
reward = self.update_reward()
else:
reward = 0
self.steps += 1
self.global_steps += 1
done = self.env.is_episode_finished()
# state is available only if the episode is still running
if not done:
self._rgb_array = self.env.get_state().screen_buffer
observation = self._process_image(self._rgb_array)
return observation, reward, done
def step(self, action):
observation, reward, done = self.sub_step(action)
return observation, reward, done, {}
def close(self):
"""Close environment"""
self.env.close()
def seed(self, seed=None):
"""Seed"""
if seed:
self.env.set_seed(seed)
def render(self, mode='human'):
"""Render frame"""
if mode == 'rgb_array':
return self._rgb_array
raise NotImplementedError