class DoomEnv(gym.Env): metadata = { 'render.modes': ['human', 'rgb_array'], 'video.frames_per_second': 35 } def __init__(self, level): self.previous_level = -1 self.level = level self.game = DoomGame() self.loader = Loader() self.doom_dir = os.path.dirname(os.path.abspath(__file__)) self._mode = 'algo' # 'algo' or 'human' self.no_render = False # To disable double rendering in human mode self.viewer = None self.is_initialized = False # Indicates that reset() has been called self.curr_seed = 0 self.lock = (DoomLock()).get_lock() # self.action_space = spaces.Discrete(43) # used to be in the old code self.action_space = spaces.MultiBinary(NUM_ACTIONS) self.allowed_actions = list(range(NUM_ACTIONS)) self.screen_height = 120 self.screen_width = 160 self.screen_resolution = ScreenResolution.RES_160X120 self.observation_space = spaces.Box(low=0, high=255, shape=(self.screen_height, self.screen_width, 3), dtype=np.uint8) self.seed() self._configure() def _configure(self, lock=None, **kwargs): if 'screen_resolution' in kwargs: logger.warn( 'Deprecated - Screen resolution must now be set using a wrapper. See documentation for details.' ) # Multiprocessing lock if lock is not None: self.lock = lock def _load_level(self): # Closing if is_initialized if self.is_initialized: self.is_initialized = False self.game.close() self.game = DoomGame() # Customizing level if getattr(self, '_customize_game', None) is not None and callable( self._customize_game): self.level = -1 self._customize_game() else: # Loading Paths if not self.is_initialized: self.game.set_vizdoom_path(self.loader.get_vizdoom_path()) self.game.set_doom_game_path(self.loader.get_freedoom_path()) # Common settings self.game.load_config( os.path.join(self.doom_dir, 'assets/%s' % DOOM_SETTINGS[self.level][CONFIG])) self.game.set_doom_scenario_path( self.loader.get_scenario_path( DOOM_SETTINGS[self.level][SCENARIO])) if DOOM_SETTINGS[self.level][MAP] != '': if RANDOMIZE_MAPS > 0 and 'labyrinth' in DOOM_SETTINGS[ self.level][CONFIG].lower(): if 'fix' in DOOM_SETTINGS[self.level][SCENARIO].lower(): # mapId = 'map%02d'%np.random.randint(1, 23) mapId = 'map%02d' % np.random.randint(4, 8) else: mapId = 'map%02d' % np.random.randint( 1, RANDOMIZE_MAPS + 1) print( '\t=> Special Config: Randomly Loading Maps. MapID = ' + mapId) self.game.set_doom_map(mapId) else: print('\t=> Default map loaded. MapID = ' + DOOM_SETTINGS[self.level][MAP]) self.game.set_doom_map(DOOM_SETTINGS[self.level][MAP]) self.game.set_doom_skill(DOOM_SETTINGS[self.level][DIFFICULTY]) self.allowed_actions = DOOM_SETTINGS[self.level][ACTIONS] self.game.set_screen_resolution(self.screen_resolution) self.previous_level = self.level self._closed = False # Algo mode if 'human' != self._mode: if NO_MONSTERS: print('\t=> Special Config: Monsters Removed.') self.game.add_game_args('-nomonsters 1') self.game self.game.set_window_visible(False) self.game.set_mode(Mode.PLAYER) self.no_render = False try: with self.lock: self.game.init() except (ViZDoomUnexpectedExitException, ViZDoomErrorException): raise error.Error( 'VizDoom exited unexpectedly. This is likely caused by a missing multiprocessing lock. ' + 'To run VizDoom across multiple processes, you need to pass a lock when you configure the env ' + '[e.g. env.configure(lock=my_multiprocessing_lock)], or create and close an env ' + 'before starting your processes [e.g. env = gym.make("DoomBasic-v0"); env.close()] to cache a ' + 'singleton lock in memory.') self._start_episode() self.is_initialized = True return self.game.get_state().screen_buffer.copy() # Human mode else: if NO_MONSTERS: print('\t=> Special Config: Monsters Removed.') self.game.add_game_args('-nomonsters 1') self.game.add_game_args('+freelook 1') self.game.set_window_visible(True) self.game.set_mode(Mode.SPECTATOR) self.no_render = True with self.lock: self.game.init() self._start_episode() self.is_initialized = True self._play_human_mode() return np.zeros(shape=self.observation_space.shape, dtype=np.uint8) def _start_episode(self): if self.curr_seed > 0: self.game.set_seed(self.curr_seed) self.curr_seed = 0 self.game.new_episode() return def _play_human_mode(self): while not self.game.is_episode_finished(): self.game.advance_action() state = self.game.get_state() total_reward = self.game.get_total_reward() info = self._get_game_variables(state.game_variables) info["TOTAL_REWARD"] = round(total_reward, 4) print('===============================') print('State: #' + str(state.number)) print('Action: \t' + str(self.game.get_last_action()) + '\t (=> only allowed actions)') print('Reward: \t' + str(self.game.get_last_reward())) print('Total Reward: \t' + str(total_reward)) print('Variables: \n' + str(info)) sleep(0.02857) # 35 fps = 0.02857 sleep between frames print('===============================') print('Done') return def old_step(self, action): """ action: a number in range 0..42 We get this from the simontudo and his predecessors, it transforms a numeric action from space Discrete(43) into a indicator action . However, we can only press one button at the same time. """ # Convert to array action_arr = np.zeros(NUM_ACTIONS, dtype=int) action_arr[action] = 1 action = action_arr assert self.is_initialized, "Doom env not reset, call .reset()" # action is a list of numbers but DoomGame.make_action expects a list of ints if len(self.allowed_actions) > 0: list_action = [ int(action[action_idx]) for action_idx in self.allowed_actions ] else: list_action = [int(x) for x in action] try: reward = self.game.make_action(list_action) state = self.game.get_state() if self.game.is_episode_finished(): info = {"TOTAL_REWARD": round(self.game.get_total_reward(), 4)} is_finished = True return np.zeros(shape=self.observation_space.shape, dtype=np.uint8), reward, is_finished, info else: info = self._get_game_variables(state.game_variables) info["TOTAL_REWARD"] = round(self.game.get_total_reward(), 4) is_finished = False return state.screen_buffer.copy(), reward, is_finished, info except vizdoom.vizdoom.ViZDoomIsNotRunningException: return np.zeros(shape=self.observation_space.shape, dtype=np.uint8), 0, True, {} def step(self, action): """ action: iterable of length 43, contains indicators of whether given buttons was pressed. Written by me. """ list_action = [int(x) for x in action] try: reward = self.game.make_action(list_action) state = self.game.get_state() if self.game.is_episode_finished(): info = {"TOTAL_REWARD": round(self.game.get_total_reward(), 4)} is_finished = True return np.zeros(shape=self.observation_space.shape, dtype=np.uint8), reward, is_finished, info else: info = self._get_game_variables(state.game_variables) info["TOTAL_REWARD"] = round(self.game.get_total_reward(), 4) is_finished = False return state.screen_buffer.copy(), reward, is_finished, info except vizdoom.vizdoom.ViZDoomIsNotRunningException: return np.zeros(shape=self.observation_space.shape, dtype=np.uint8), 0, True, {} def reset(self): if self.is_initialized and not self._closed: self._start_episode() screen_buffer = self.game.get_state().screen_buffer if screen_buffer is None: raise error.Error( 'VizDoom incorrectly initiated. This is likely caused by a missing multiprocessing lock. ' + 'To run VizDoom across multiple processes, you need to pass a lock when you configure the env ' + '[e.g. env.configure(lock=my_multiprocessing_lock)], or create and close an env ' + 'before starting your processes [e.g. env = gym.make("DoomBasic-v0"); env.close()] to cache a ' + 'singleton lock in memory.') return screen_buffer.copy() else: return self._load_level() def render(self, mode='human', close=False): if close: if self.viewer is not None: self.viewer.close() self.viewer = None # If we don't None out this reference pyglet becomes unhappy return try: if 'human' == mode and self.no_render: return state = self.game.get_state() img = state.screen_buffer # VizDoom returns None if the episode is finished, let's make it # an empty image so the recorder doesn't stop if img is None: img = np.zeros(shape=self.observation_space.shape, dtype=np.uint8) if mode == 'rgb_array': return img elif mode is 'human': from gym.envs.classic_control import rendering if self.viewer is None: self.viewer = rendering.SimpleImageViewer() self.viewer.imshow(img) except vizdoom.vizdoom.ViZDoomIsNotRunningException: return np.zeros(shape=self.observation_space.shape, dtype=np.uint8) except AttributeError: return np.zeros(shape=self.observation_space.shape, dtype=np.uint8) def close(self): # Lock required for VizDoom to close processes properly with self.lock: self.game.close() def seed(self, seed=None): self.curr_seed = seeding.hash_seed(seed) % 2**32 return [self.curr_seed] def _get_game_variables(self, state_variables): info = {"LEVEL": self.level} if state_variables is None: return info info['KILLCOUNT'] = state_variables[0] info['ITEMCOUNT'] = state_variables[1] info['SECRETCOUNT'] = state_variables[2] info['FRAGCOUNT'] = state_variables[3] info['HEALTH'] = state_variables[4] info['ARMOR'] = state_variables[5] info['DEAD'] = state_variables[6] info['ON_GROUND'] = state_variables[7] info['ATTACK_READY'] = state_variables[8] info['ALTATTACK_READY'] = state_variables[9] info['SELECTED_WEAPON'] = state_variables[10] info['SELECTED_WEAPON_AMMO'] = state_variables[11] info['AMMO1'] = state_variables[12] info['AMMO2'] = state_variables[13] info['AMMO3'] = state_variables[14] info['AMMO4'] = state_variables[15] info['AMMO5'] = state_variables[16] info['AMMO6'] = state_variables[17] info['AMMO7'] = state_variables[18] info['AMMO8'] = state_variables[19] info['AMMO9'] = state_variables[20] info['AMMO0'] = state_variables[21] return info
class Game(object): def __init__(self, scenario, action_builder, reward_values=None, score_variable='FRAGCOUNT', freedoom=True, screen_resolution='RES_400X225', screen_format='CRCGCB', use_screen_buffer=True, use_depth_buffer=False, labels_mapping='', game_features='', mode='PLAYER', player_rank=0, players_per_game=1, render_hud=False, render_minimal_hud=False, render_crosshair=True, render_weapon=True, render_decals=False, render_particles=False, render_effects_sprites=False, respawn_protect=True, spawn_farthest=True, freelook=False, name='Arnold', color=0, visible=False, n_bots=0, use_scripted_marines=None, doom_skill=2): """ Create a new game. score_variable: indicates in which game variable the user score is stored. by default it's in FRAGCOUNT, but the score in ACS against built-in AI bots can be stored in USER1, USER2, etc. render_decals: marks on the walls render_particles: particles like for impacts / traces render_effects_sprites: gun puffs / blood splats color: 0 - green, 1 - gray, 2 - brown, 3 - red, 4 - light gray, 5 - light brown, 6 - light red, 7 - light blue """ # game resources game_filename = '%s.wad' % ('freedoom2' if freedoom else 'Doom2') self.scenario_path = os.path.join(RESOURCES_DIR, 'scenarios', '%s.wad' % scenario) self.game_path = os.path.join(RESOURCES_DIR, game_filename) # check parameters assert os.path.isfile(self.scenario_path) assert os.path.isfile(self.game_path) assert hasattr(GameVariable, score_variable) assert hasattr(ScreenResolution, screen_resolution) assert hasattr(ScreenFormat, screen_format) assert use_screen_buffer or use_depth_buffer assert hasattr(Mode, mode) assert not (render_minimal_hud and not render_hud) assert len(name.strip()) > 0 and color in range(8) assert n_bots >= 0 assert (type(use_scripted_marines) is bool or use_scripted_marines is None and n_bots == 0) assert 0 <= doom_skill <= 4 assert 0 < players_per_game assert 0 <= player_rank # action builder self.action_builder = action_builder # add the score variable to the game variables list self.score_variable = score_variable game_variables.append(('score', getattr(GameVariable, score_variable))) self.player_rank = player_rank self.players_per_game = players_per_game # screen buffer / depth buffer / labels buffer / mode self.screen_resolution = screen_resolution self.screen_format = screen_format self.use_screen_buffer = use_screen_buffer self.use_depth_buffer = use_depth_buffer self.labels_mapping = parse_labels_mapping(labels_mapping) self.game_features = parse_game_features(game_features) self.use_labels_buffer = self.labels_mapping is not None self.use_game_features = any(self.game_features) self.mode = mode # rendering options self.render_hud = render_hud self.render_minimal_hud = render_minimal_hud self.render_crosshair = render_crosshair self.render_weapon = render_weapon self.render_decals = render_decals self.render_particles = render_particles self.render_effects_sprites = render_effects_sprites # respawn invincibility / distance self.respawn_protect = respawn_protect self.spawn_farthest = spawn_farthest # freelook / agent name / agent color self.freelook = freelook self.name = name.strip() self.color = color # window visibility self.visible = visible # actor reward self.reward_builder = RewardBuilder(self, reward_values) # game statistics self.stat_keys = [ 'kills', 'deaths', 'suicides', 'frags', 'k/d', 'medikits', 'armors', 'pistol', 'shotgun', 'chaingun', 'rocketlauncher', 'plasmarifle', 'bfg9000', 'bullets', 'shells', 'rockets', 'cells' ] self.statistics = {} # number of bots in the game self.n_bots = n_bots self.use_scripted_marines = use_scripted_marines # doom skill self.doom_skill = doom_skill # manual control self.count_non_forward_actions = 0 self.count_non_turn_actions = 0 def update_game_variables(self): """ Check and update game variables. """ # read game variables new_v = {k: self.game.get_game_variable(v) for k, v in game_variables} assert all(v.is_integer() or k[-2:] in ['_x', '_y', '_z'] for k, v in new_v.items()) new_v = { k: (int(v) if v.is_integer() else float(v)) for k, v in new_v.items() } health = new_v['health'] armor = new_v['armor'] sel_weapon = new_v['sel_weapon'] sel_ammo = new_v['sel_ammo'] bullets = new_v['bullets'] shells = new_v['shells'] rockets = new_v['rockets'] cells = new_v['cells'] fist = new_v['fist'] pistol = new_v['pistol'] shotgun = new_v['shotgun'] chaingun = new_v['chaingun'] rocketlauncher = new_v['rocketlauncher'] plasmarifle = new_v['plasmarifle'] bfg9000 = new_v['bfg9000'] # check game variables if sel_weapon == -1: logger.warning("SELECTED WEAPON is -1!") new_v['sel_weapon'] = 1 sel_weapon = 1 if sel_ammo == -1: logger.warning("SELECTED AMMO is -1!") new_v['sel_ammo'] = 0 sel_ammo = 0 assert sel_weapon in range(1, 8), sel_weapon assert sel_ammo >= 0, sel_ammo assert all(x in [0, 1] for x in [ fist, pistol, shotgun, chaingun, rocketlauncher, plasmarifle, bfg9000 ]) assert 0 <= health <= 200 or health < 0 and self.game.is_player_dead() assert 0 <= armor <= 200, (health, armor) assert 0 <= bullets <= 200 and 0 <= shells <= 50 assert 0 <= rockets <= 50 and 0 <= cells <= 300 # fist if sel_weapon == 1: assert sel_ammo == 0 # pistol elif sel_weapon == 2: assert pistol and sel_ammo == bullets # shotgun elif sel_weapon == 3: assert shotgun and sel_ammo == shells # chaingun elif sel_weapon == 4: assert chaingun and sel_ammo == bullets # rocket launcher elif sel_weapon == 5: assert rocketlauncher and sel_ammo == rockets # plasma rifle elif sel_weapon == 6: assert plasmarifle and sel_ammo == cells # BFG9000 elif sel_weapon == 7: assert bfg9000 and sel_ammo == cells # update actor properties self.prev_properties = self.properties self.properties = new_v def update_statistics_and_reward(self, action): """ Update statistics of the current game based on the previous and the current properties, and create a reward. """ stats = self.statistics[self.map_id] # reset reward self.reward_builder.reset() # we need to know the current and previous properties assert self.prev_properties is not None and self.properties is not None # distance moving_forward = action[self.mapping['MOVE_FORWARD']] turn_left = action[self.mapping['TURN_LEFT']] turn_right = action[self.mapping['TURN_RIGHT']] if moving_forward and not (turn_left or turn_right): diff_x = self.properties['position_x'] - self.prev_properties[ 'position_x'] diff_y = self.properties['position_y'] - self.prev_properties[ 'position_y'] distance = math.sqrt(diff_x**2 + diff_y**2) self.reward_builder.distance(distance) # kill d = self.properties['score'] - self.prev_properties['score'] if d > 0: self.reward_builder.kill(d) stats['kills'] += d for _ in range(int(d)): self.log('Kill') # death if self.game.is_player_dead(): self.reward_builder.death() stats['deaths'] += 1 self.log('Dead') # suicide if self.properties['frag_count'] < self.prev_properties['frag_count']: self.reward_builder.suicide() stats['suicides'] += 1 self.log('Suicide') # found / lost health d = self.properties['health'] - self.prev_properties['health'] if d != 0: if d > 0: self.reward_builder.medikit(d) stats['medikits'] += 1 else: self.reward_builder.injured(d) self.log('%s health (%i -> %i)' % ( 'Found' if d > 0 else 'Lost', self.prev_properties['health'], self.properties['health'], )) # found / lost armor d = self.properties['armor'] - self.prev_properties['armor'] if d != 0: if d > 0: self.reward_builder.armor() stats['armors'] += 1 self.log('%s armor (%i -> %i)' % ( 'Found' if d > 0 else 'Lost', self.prev_properties['armor'], self.properties['armor'], )) # change weapon if self.properties['sel_weapon'] != self.prev_properties['sel_weapon']: self.log('Switched weapon: %s -> %s' % ( WEAPON_NAMES[self.prev_properties['sel_weapon']], WEAPON_NAMES[self.properties['sel_weapon']], )) # found weapon for i, weapon in enumerate([ 'pistol', 'shotgun', 'chaingun', 'rocketlauncher', 'plasmarifle', 'bfg9000' ]): if self.prev_properties[weapon] == self.properties[weapon]: continue # assert(self.prev_properties[weapon] == 0 and # TODO check # self.properties[weapon] == 1), (weapon, self.prev_properties[weapon], self.properties[weapon]) self.reward_builder.weapon() stats[weapon] += 1 self.log('Found weapon: %s' % WEAPON_NAMES[i + 1]) # found / lost ammo for ammo in ['bullets', 'shells', 'rockets', 'cells']: d = self.properties[ammo] - self.prev_properties[ammo] if d != 0: if d > 0: self.reward_builder.ammo() stats[ammo] += 1 else: self.reward_builder.use_ammo() self.log('%s ammo: %s (%i -> %i)' % ('Found' if d > 0 else 'Lost', ammo, self.prev_properties[ammo], self.properties[ammo])) def log(self, message): """ Log the game event. During training, we don't want to display events. """ if self.log_events: logger.info(message) def start(self, map_id, episode_time=None, manual_control=False, log_events=False): """ Start the game. If `episode_time` is given, the game will end after the specified time. """ assert type(manual_control) is bool self.manual_control = manual_control # Save statistics for this map self.statistics[map_id] = {k: 0 for k in self.stat_keys} # Episode time self.episode_time = episode_time # initialize the game self.game = DoomGame() self.game.set_doom_scenario_path(self.scenario_path) self.game.set_doom_game_path(self.game_path) # map assert map_id > 0 self.map_id = map_id self.game.set_doom_map("map%02i" % map_id) # time limit if episode_time is not None: self.game.set_episode_timeout(int(35 * episode_time)) # log events that happen during the game (useful for testing) self.log_events = log_events # game parameters args = [] # host / server if self.players_per_game > 1: port = 5092 + self.player_rank // self.players_per_game if self.player_rank % self.players_per_game == 0: args.append('-host %i -port %i' % (self.players_per_game, port)) else: args.append('-join 127.0.0.1:%i' % port) else: args.append('-host 1') # screen buffer / depth buffer / labels buffer / mode screen_resolution = getattr(ScreenResolution, self.screen_resolution) self.game.set_screen_resolution(screen_resolution) self.game.set_screen_format(getattr(ScreenFormat, self.screen_format)) self.game.set_depth_buffer_enabled(self.use_depth_buffer) self.game.set_labels_buffer_enabled(self.use_labels_buffer or self.use_game_features) self.game.set_mode(getattr(Mode, self.mode)) # rendering options self.game.set_render_hud(self.render_hud) self.game.set_render_minimal_hud(self.render_minimal_hud) self.game.set_render_crosshair(self.render_crosshair) self.game.set_render_weapon(self.render_weapon) self.game.set_render_decals(self.render_decals) self.game.set_render_particles(self.render_particles) self.game.set_render_effects_sprites(self.render_effects_sprites) # deathmatch mode # players will respawn automatically after they die # autoaim is disabled for all players args.append('-deathmatch') args.append('+sv_forcerespawn 1') args.append('+sv_noautoaim 1') # respawn invincibility / distance # players will be invulnerable for two second after spawning # players will be spawned as far as possible from any other players args.append('+sv_respawnprotect %i' % self.respawn_protect) args.append('+sv_spawnfarthest %i' % self.spawn_farthest) # freelook / agent name / agent color args.append('+freelook %i' % (1 if self.freelook else 0)) args.append('+name %s' % self.name) args.append('+colorset %i' % self.color) # enable the cheat system (so that we can still # send commands to the game in self-play mode) args.append('+sv_cheats 1') # load parameters self.args = args for arg in args: self.game.add_game_args(arg) # window visibility self.game.set_window_visible(self.visible) # available buttons self.mapping = add_buttons(self.game, self.action_builder.available_buttons) # doom skill (https://zdoom.org/wiki/GameSkill) self.game.set_doom_skill(self.doom_skill + 1) # start the game self.game.init() # initialize the game after player spawns self.initialize_game() def reset(self): """ Reset the game if necessary. This can be because: - we reach the end of an episode (we restart the game) - because the agent is dead (we make it respawn) """ self.count_non_forward_actions = 0 # if the player is dead if self.is_player_dead(): # respawn it (deathmatch mode) if self.episode_time is None: self.respawn_player() # or reset the episode (episode ends when the agent dies) else: self.new_episode() # start a new episode if it is finished if self.is_episode_finished(): self.new_episode() # deal with a ViZDoom issue while self.is_player_dead(): logger.warning('Player %i is still dead after respawn.' % self.params.player_rank) self.respawn_player() def update_bots(self): """ Add built-in AI bots. There are two types of AI: built-in AI and ScriptedMarines. """ # only the host takes care of the bots if self.player_rank % self.players_per_game != 0: return if self.use_scripted_marines: command = "pukename set_value always 2 %i" % self.n_bots self.game.send_game_command(command) else: self.game.send_game_command("removebots") for _ in range(self.n_bots): self.game.send_game_command("addbot") def is_player_dead(self): """ Detect whether the player is dead. """ return self.game.is_player_dead() def is_episode_finished(self): """ Return whether the episode is finished. This should only be the case after the episode timeout. """ return self.game.is_episode_finished() def is_final(self): """ Return whether the game is in a final state. """ return self.is_player_dead() or self.is_episode_finished() def new_episode(self): """ Start a new episode. """ assert self.is_episode_finished() or self.is_player_dead() self.game.new_episode() self.log('New episode') self.initialize_game() def respawn_player(self): """ Respawn the player on death. """ assert self.is_player_dead() self.game.respawn_player() self.log('Respawn player') self.initialize_game() def initialize_game(self): """ Initialize the game after the player spawns / respawns. Be sure that properties from the previous life are not considered in this one. """ # generate buffers game_state = self.game.get_state() self._screen_buffer = game_state.screen_buffer self._depth_buffer = game_state.depth_buffer self._labels_buffer = game_state.labels_buffer self._labels = game_state.labels # actor properties self.prev_properties = None self.properties = None # advance a few steps to avoid bugs due # to initial weapon changes in ACS self.game.advance_action(SKIP_INITIAL_ACTIONS) self.update_game_variables() # if there are bots in the game, and if this is a new game self.update_bots() def randomize_textures(self, randomize): """ Randomize the textures of the map. """ assert type(randomize) is bool randomize = 1 if randomize else 0 self.game.send_game_command("pukename set_value always 4 %i" % randomize) def init_bots_health(self, health): """ Initial bots health. """ assert self.use_scripted_marines or health == 100 assert 0 < health <= 100 self.game.send_game_command("pukename set_value always 5 %i" % health) def make_action(self, action, frame_skip=1, sleep=None): """ Make an action. If `sleep` is given, the network will wait `sleep` seconds between each action. """ assert frame_skip >= 1 # convert selected action to the ViZDoom action format action = self.action_builder.get_action(action) # select agent favorite weapon for weapon_name, weapon_ammo, weapon_id in WEAPONS_PREFERENCES: min_ammo = 40 if weapon_name == 'bfg9000' else 1 if self.properties[weapon_name] > 0 and self.properties[ weapon_ammo] >= min_ammo: if self.properties['sel_weapon'] != weapon_id: # action = ([False] * self.mapping['SELECT_WEAPON%i' % weapon_id]) + [True] switch_action = ( [False] * self.mapping['SELECT_WEAPON%i' % weapon_id]) + [True] action = action + switch_action[len(action):] self.log("Manual weapon change: %s -> %s" % (WEAPON_NAMES[self.properties['sel_weapon']], weapon_name)) break if action[self.mapping['MOVE_FORWARD']]: self.count_non_forward_actions = 0 else: self.count_non_forward_actions += 1 if action[self.mapping['TURN_LEFT']] or action[ self.mapping['TURN_RIGHT']]: self.count_non_turn_actions = 0 else: self.count_non_turn_actions += 1 if self.manual_control and (self.count_non_forward_actions >= 30 or self.count_non_turn_actions >= 60): manual_action = [False] * len(action) manual_action[self.mapping['TURN_RIGHT']] = True manual_action[self.mapping['SPEED']] = True if self.count_non_forward_actions >= 30: manual_action[self.mapping['MOVE_FORWARD']] = True manual_repeat = 40 self.count_non_forward_actions = 0 self.count_non_turn_actions = 0 else: manual_action = None # if we are visualizing the experiment, show all the frames one by one if self.visible: if manual_action is not None: logger.warning('Activated manual control') for _ in range(manual_repeat): self.game.make_action(manual_action) else: for _ in range(frame_skip): self.game.make_action(action) # death or episode finished if self.is_player_dead() or self.is_episode_finished(): break # sleep for smooth visualization if sleep is not None: time.sleep(sleep) else: if manual_action is not None: logger.warning('Activated manual control') self.game.make_action(manual_action, manual_repeat) else: self.game.make_action(action, frame_skip) # generate buffers game_state = self.game.get_state() if game_state is not None: self._screen_buffer = game_state.screen_buffer self._depth_buffer = game_state.depth_buffer self._labels_buffer = game_state.labels_buffer self._labels = game_state.labels # update game variables / statistics rewards self.update_game_variables() self.update_statistics_and_reward(action) @property def reward(self): """ Return the reward value. """ return self.reward_builder.reward def close(self): """ Close the current game. """ self.game.close() def print_statistics(self, eval_time=None): """ Print agent statistics. If `map_id` is given, statistics are given for the specified map only. Otherwise, statistics are given for all maps, with a summary. """ if 'all' in self.statistics: del self.statistics['all'] map_ids = sorted(self.statistics.keys()) if len(map_ids) == 0: logger.info("No statistics to show!") return for v in self.statistics.values(): assert set(self.stat_keys) == set(v.keys()) # sum the results on all maps for global statistics self.statistics['all'] = { k: sum(v[k] for v in self.statistics.values()) for k in self.stat_keys } # number of frags (kills - suicides) # 100% accurate if the number of frags is given by 'FRAGCOUNT' # almost 100% accurate if it is based on an internal ACS variable for v in self.statistics.values(): v['frags'] = v['kills'] - v['suicides'] # number of frags per minutes (with and without respawn time) if eval_time is not None: assert eval_time % 60 == 0 for k, v in self.statistics.items(): eval_minutes = eval_time / 60 if k == 'all': eval_minutes *= (len(self.statistics) - 1) respawn_time = (v['deaths'] * RESPAWN_SECONDS * 1.0 / 60) v['frags_pm'] = v['frags'] * 1.0 / eval_minutes v['frags_pm_r'] = v['frags'] * 1.0 / (eval_minutes + respawn_time) # Kills / Deaths # 100% accurate if the number of kills is given by an ACS variable # almost 100% accurate if it is based on 'FRAGCOUNT' for v in self.statistics.values(): v['k/d'] = v['kills'] * 1.0 / max(1, v['deaths']) # statistics to log log_lines = [ [''] + ['Map%02i' % i for i in map_ids] + ['All'], ('Kills', 'kills'), ('Deaths', 'deaths'), ('Suicides', 'suicides'), ('Frags', 'frags'), ('Frags/m', 'frags_pm'), ('Frags/m (r)', 'frags_pm_r'), ('K/D', 'k/d'), None, ('Medikits', 'medikits'), ('Armors', 'armors'), ('SuperShotgun', 'shotgun'), ('Chaingun', 'chaingun'), ('RocketLauncher', 'rocketlauncher'), ('PlasmaRifle', 'plasmarifle'), ('BFG9000', 'bfg9000'), ('Bullets', 'bullets'), ('Shells', 'shells'), ('Rockets', 'rockets'), ('Cells', 'cells'), ] # only show statistics on all maps if there is more than one map if len(map_ids) > 1: map_ids.append('all') logger.info('*************** Game statistics summary ***************') log_pattern = '{: >15}' + ('{: >8}' * len(map_ids)) for line in log_lines: if line is None: logger.info('') else: if type(line) is tuple: assert len(line) == 2 name, k = line if k in ['frags_pm', 'frags_pm_r'] and eval_time is None: continue line = ['%s:' % name] line += [self.statistics[map_id][k] for map_id in map_ids] else: assert type(line) is list line = line[:len(map_ids) + 1] line = ['%.3f' % x if type(x) is float else x for x in line] logger.info(log_pattern.format(*line)) def observe_state(self, params, last_states): """ Observe the current state of the game. """ # read game state screen, game_features = process_buffers(self, params) variables = [self.properties[x[0]] for x in params.game_variables] last_states.append(GameState(screen, variables, game_features)) # update most recent states if len(last_states) == 1: last_states.extend([last_states[0]] * (params.hist_size - 1)) else: assert len(last_states) == params.hist_size + 1 del last_states[0] # return the screen and the game features return screen, game_features
class Game(object): def __init__(self, scenario, action_builder, score_variable='FRAGCOUNT', freedoom=True, screen_resolution='RES_400X225', screen_format='CRCGCB', use_screen_buffer=True, use_depth_buffer=False, labels_mapping='', game_features='', mode='ASYNC_PLAYER', render_hud=False, render_minimal_hud=False, render_crosshair=True, render_weapon=True, render_decals=False, render_particles=False, render_effects_sprites=False, respawn_protect=True, spawn_farthest=True, freelook=False, name='LUBAN', color=0, visible=False, n_bots=0, use_scripted_marines=None, doom_skill=2): """ Create a new game. score_variable: indicates in which game variable the user score is stored. by default it's in FRAGCOUNT, but the score in ACS against built-in AI bots can be stored in USER1, USER2, etc. render_decals: marks on the walls render_particles: particles like for impacts / traces render_effects_sprites: gun puffs / blood splats color: 0 - green, 1 - gray, 2 - brown, 3 - red, 4 - light gray, 5 - light brown, 6 - light red, 7 - light blue """ # game resources game_filename = '%s.wad' % ('freedoom2' if freedoom else 'Doom2') self.scenario_path = os.path.join(RESOURCES_DIR, '%s.wad' % scenario) self.game_path = os.path.join(RESOURCES_DIR, game_filename) print(self.scenario_path) print(self.game_path) # check parameters assert os.path.isfile(self.scenario_path) assert os.path.isfile(self.game_path) assert hasattr(GameVariable, score_variable) assert hasattr(ScreenResolution, screen_resolution) assert hasattr(ScreenFormat, screen_format) assert use_screen_buffer or use_depth_buffer assert hasattr(Mode, mode) assert len(name.strip()) > 0 and color in range(8) assert n_bots >= 0 assert (type(use_scripted_marines) is bool or use_scripted_marines is None and n_bots == 0) assert 0 <= doom_skill <= 4 # action builder self.action_builder = action_builder # add the score variable to the game variables list self.score_variable = score_variable game_variables.append(('score', getattr(GameVariable, score_variable))) # screen buffer / depth buffer / labels buffer / mode self.screen_resolution = screen_resolution self.screen_format = screen_format self.use_screen_buffer = use_screen_buffer self.use_depth_buffer = use_depth_buffer self.labels_mapping = parse_labels_mapping(labels_mapping) self.game_features = parse_game_features(game_features) self.use_labels_buffer = self.labels_mapping is not None self.use_game_features = any(self.game_features) self.mode = mode # rendering options self.render_hud = render_hud self.render_minimal_hud = render_minimal_hud self.render_crosshair = render_crosshair self.render_weapon = render_weapon self.render_decals = render_decals self.render_particles = render_particles self.render_effects_sprites = render_effects_sprites # respawn invincibility / distance self.respawn_protect = respawn_protect self.spawn_farthest = spawn_farthest # freelook / agent name / agent color self.freelook = freelook self.name = name.strip() self.color = color # window visibility self.visible = visible # game statistics self.stat_keys = [ 'distance', 'kills', 'deaths', 'suicides', 'frags', 'k/d', 'medikits', 'armors', 'pistol', 'shotgun', 'chaingun', 'rocketlauncher', 'plasmarifle', 'bfg9000', 'bullets', 'shells', 'rockets', 'cells' ] self.statistics = {} # number of bots in the game self.n_bots = n_bots self.use_scripted_marines = use_scripted_marines # doom skill self.doom_skill = doom_skill # manual control self.count_non_forward_actions = 0 self.count_non_turn_actions = 0 def update_game_variables(self): """ Check and update game variables. """ # read game variables new_v = {k: self.game.get_game_variable(v) for k, v in game_variables} assert all(v.is_integer() or k[-2:] in ['_x', '_y', '_z'] for k, v in new_v.items()) new_v = { k: (int(v) if v.is_integer() else float(v)) for k, v in new_v.items() } health = new_v['health'] armor = new_v['armor'] sel_weapon = new_v['sel_weapon'] sel_ammo = new_v['sel_ammo'] bullets = new_v['bullets'] shells = new_v['shells'] rockets = new_v['rockets'] cells = new_v['cells'] fist = new_v['fist'] pistol = new_v['pistol'] shotgun = new_v['shotgun'] chaingun = new_v['chaingun'] rocketlauncher = new_v['rocketlauncher'] plasmarifle = new_v['plasmarifle'] bfg9000 = new_v['bfg9000'] # check game variables if sel_weapon == -1: new_v['sel_weapon'] = 1 sel_weapon = 1 if sel_ammo == -1: new_v['sel_ammo'] = 0 sel_ammo = 0 assert sel_weapon in range(1, 8), sel_weapon assert sel_ammo >= 0, sel_ammo assert all(x in [0, 1] for x in [ fist, pistol, shotgun, chaingun, rocketlauncher, plasmarifle, bfg9000 ]) assert 0 <= health <= 200 or health < 0 and self.game.is_player_dead() assert 0 <= armor <= 200, (health, armor) assert 0 <= bullets <= 200 and 0 <= shells <= 50 assert 0 <= rockets <= 50 and 0 <= cells <= 300 # fist if sel_weapon == 1: assert sel_ammo == 0 # pistol elif sel_weapon == 2: assert pistol and sel_ammo == bullets # shotgun elif sel_weapon == 3: assert shotgun and sel_ammo == shells # chaingun elif sel_weapon == 4: assert chaingun and sel_ammo == bullets # rocket launcher elif sel_weapon == 5: assert rocketlauncher and sel_ammo == rockets # plasma rifle elif sel_weapon == 6: assert plasmarifle and sel_ammo == cells # BFG9000 elif sel_weapon == 7: assert bfg9000 and sel_ammo == cells # update actor properties self.prev_properties = self.properties self.properties = new_v def update_statistics(self, action): """ Update statistics of the current game based on the previous and the current properties for evaluating the agent performance. """ stats = self.statistics[self.map_id] # we need to know the current and previous properties assert self.prev_properties is not None and self.properties is not None # distance moving_forward = action[self.mapping['MOVE_FORWARD']] turn_left = action[self.mapping['TURN_LEFT']] turn_right = action[self.mapping['TURN_RIGHT']] if moving_forward and not (turn_left or turn_right): diff_x = self.properties['position_x'] - self.prev_properties[ 'position_x'] diff_y = self.properties['position_y'] - self.prev_properties[ 'position_y'] distance = math.sqrt(diff_x**2 + diff_y**2) stats['distance'] += distance # kill d = self.properties['score'] - self.prev_properties['score'] if d > 0: stats['kills'] += d # death if self.game.is_player_dead(): stats['deaths'] += 1 # suicide if self.properties['frag_count'] < self.prev_properties['frag_count']: stats['suicides'] += 1 # found / lost health d = self.properties['health'] - self.prev_properties['health'] if d != 0: if d > 0: stats['medikits'] += 1 # found / lost armor d = self.properties['armor'] - self.prev_properties['armor'] if d != 0: if d > 0: stats['armors'] += 1 # found weapon for i, weapon in enumerate([ 'pistol', 'shotgun', 'chaingun', 'rocketlauncher', 'plasmarifle', 'bfg9000' ]): if self.prev_properties[weapon] == self.properties[weapon]: continue stats[weapon] += 1 # found / lost ammo for ammo in ['bullets', 'shells', 'rockets', 'cells']: d = self.properties[ammo] - self.prev_properties[ammo] if d != 0: if d > 0: stats[ammo] += 1 def start(self, map_id, episode_time=None, manual_control=False): """ Start the game. If `episode_time` is given, the game will end after the specified time. """ assert type(manual_control) is bool self.manual_control = manual_control # Save statistics for this map self.statistics[map_id] = {k: 0 for k in self.stat_keys} # Episode time self.episode_time = episode_time # initialize the game self.game = DoomGame() self.game.set_doom_scenario_path(self.scenario_path) self.game.set_doom_game_path(self.game_path) # map assert map_id > 0 self.map_id = map_id self.game.set_doom_map("map%02i" % map_id) # time limit if episode_time is not None: self.game.set_episode_timeout(int(35 * episode_time)) # game parameters args = [] # host / server args.append('-host 1') # screen buffer / depth buffer / labels buffer / mode screen_resolution = getattr(ScreenResolution, self.screen_resolution) self.game.set_screen_resolution(screen_resolution) self.game.set_screen_format(getattr(ScreenFormat, self.screen_format)) self.game.set_depth_buffer_enabled(self.use_depth_buffer) self.game.set_labels_buffer_enabled(self.use_labels_buffer or self.use_game_features) self.game.set_mode(getattr(Mode, self.mode)) # rendering options self.game.set_render_hud(self.render_hud) self.game.set_render_minimal_hud(self.render_minimal_hud) self.game.set_render_crosshair(self.render_crosshair) self.game.set_render_weapon(self.render_weapon) self.game.set_render_decals(self.render_decals) self.game.set_render_particles(self.render_particles) self.game.set_render_effects_sprites(self.render_effects_sprites) # deathmatch mode # players will respawn automatically after they die # autoaim is disabled for all players args.append('-deathmatch') args.append('+sv_forcerespawn 1') args.append('+sv_noautoaim 1') # respawn invincibility / distance # players will be invulnerable for two second after spawning # players will be spawned as far as possible from any other players args.append('+sv_respawnprotect %i' % self.respawn_protect) args.append('+sv_spawnfarthest %i' % self.spawn_farthest) # freelook / agent name / agent color args.append('+freelook %i' % (1 if self.freelook else 0)) args.append('+name %s' % self.name) args.append('+colorset %i' % self.color) # enable the cheat system (so that we can still # send commands to the game in self-play mode) args.append('+sv_cheats 1') # load parameters self.args = args for arg in args: self.game.add_game_args(arg) # window visibility self.game.set_window_visible(self.visible) # available buttons self.mapping = add_buttons(self.game, self.action_builder.available_buttons) # doom skill self.game.set_doom_skill(self.doom_skill + 1) # start the game self.game.init() # initialize the game after player spawns self.initialize_game() def update_bots(self): """ Add built-in AI bots. There are two types of AI: built-in AI and ScriptedMarines. """ # only the host takes care of the bots if self.use_scripted_marines: command = "pukename set_value always 2 %i" % self.n_bots self.game.send_game_command(command) else: self.game.send_game_command("removebots") for _ in range(self.n_bots): self.game.send_game_command("addbot") def is_player_dead(self): """ Detect whether the player is dead. """ return self.game.is_player_dead() def is_episode_finished(self): """ Return whether the episode is finished. This should only be the case after the episode timeout. """ return self.game.is_episode_finished() def is_final(self): """ Return whether the game is in a final state. """ return self.is_player_dead() or self.is_episode_finished() def new_episode(self): """ Start a new episode. """ assert self.is_episode_finished() or self.is_player_dead() self.game.new_episode() self.initialize_game() def respawn_player(self): """ Respawn the player on death. """ assert self.is_player_dead() self.game.respawn_player() self.initialize_game() def initialize_game(self): """ Initialize the game after the player spawns / respawns. Be sure that properties from the previous life are not considered in this one. """ # generate buffers game_state = self.game.get_state() self._screen_buffer = game_state.screen_buffer self._depth_buffer = game_state.depth_buffer self._labels_buffer = game_state.labels_buffer self._labels = game_state.labels # actor properties self.prev_properties = None self.properties = None # advance a few steps to avoid bugs due to initial weapon changes in ACS self.game.advance_action(SKIP_INITIAL_ACTIONS) self.update_game_variables() # if there are bots in the game, and if this is a new game self.update_bots() def randomize_textures(self, randomize): """ Randomize the textures of the map. """ assert type(randomize) is bool randomize = 1 if randomize else 0 self.game.send_game_command("pukename set_value always 4 %i" % randomize) def init_bots_health(self, health): """ Initial bots health. """ assert self.use_scripted_marines or health == 100 assert 0 < health <= 100 self.game.send_game_command("pukename set_value always 5 %i" % health) def make_action(self, action, frame_skip=1, sleep=None): """ Make an action. If `sleep` is given, the network will wait `sleep` seconds between each action. """ assert frame_skip >= 1 # convert selected action to the ViZDoom action format action = self.action_builder.get_action(action) # select agent favorite weapon for weapon_name, weapon_ammo, weapon_id in WEAPONS_PREFERENCES: min_ammo = 40 if weapon_name == 'bfg9000' else 1 if self.properties[weapon_name] > 0 and self.properties[ weapon_ammo] >= min_ammo: if self.properties['sel_weapon'] != weapon_id: switch_action = ( [False] * self.mapping['SELECT_WEAPON%i' % weapon_id]) + [True] action = action + switch_action[len(action):] break if action[self.mapping['MOVE_FORWARD']]: self.count_non_forward_actions = 0 else: self.count_non_forward_actions += 1 if action[self.mapping['TURN_LEFT']] or action[ self.mapping['TURN_RIGHT']]: self.count_non_turn_actions = 0 else: self.count_non_turn_actions += 1 if self.manual_control and (self.count_non_forward_actions >= 30 or self.count_non_turn_actions >= 60): manual_action = [False] * len(action) manual_action[self.mapping['TURN_RIGHT']] = True manual_action[self.mapping['SPEED']] = True if self.count_non_forward_actions >= 30: manual_action[self.mapping['MOVE_FORWARD']] = True manual_repeat = 40 self.count_non_forward_actions = 0 self.count_non_turn_actions = 0 else: manual_action = None # if we are visualizing the experiment, show all the frames one by one if self.visible: if manual_action is not None: for _ in range(manual_repeat): self.game.make_action(manual_action) else: for _ in range(frame_skip): self.game.make_action(action) # death or episode finished if self.is_player_dead() or self.is_episode_finished(): break # sleep for smooth visualization if sleep is not None: time.sleep(sleep) else: if manual_action is not None: self.game.make_action(manual_action, manual_repeat) else: self.game.make_action(action, frame_skip) # generate buffers game_state = self.game.get_state() if game_state is not None: self._screen_buffer = game_state.screen_buffer self._depth_buffer = game_state.depth_buffer self._labels_buffer = game_state.labels_buffer self._labels = game_state.labels # update game variables / statistics rewards self.update_game_variables() self.update_statistics(action) def close(self): """ Close the current game. """ self.game.close() def print_statistics(self, eval_time=None): """ Print agent statistics. If `map_id` is is given, statistics are given for the specified map only. """ map_ids = self.statistics.keys() assert len(map_ids) == 1 for v in self.statistics.values(): assert set(self.stat_keys) == set(v.keys()) # number of frags (kills - suicides) # 100% accurate if the number of frags is given by 'FRAGCOUNT' # almost 100% accurate if it is based on an internal ACS variable for v in self.statistics.values(): v['frags'] = v['kills'] - v['suicides'] # Kills / Deaths # 100% accurate if the number of kills is given by an ACS variable # almost 100% accurate if it is based on 'FRAGCOUNT' for v in self.statistics.values(): v['k/d'] = v['kills'] * 1.0 / max(1, v['deaths']) print("******************Game statistics summary********************") print("Map%02d" % self.map_id) for item in self.stat_keys: print(item + ":\t%d" % self.statistics[self.map_id][item]) def observe_state(self, params, last_states): """ Observe the current state of the game. """ # read game state screen, game_features = process_buffers(self, params) variables = [self.properties[x[0]] for x in params.game_variables] last_states.append(GameState(screen, variables, game_features)) # update most recent states if len(last_states) == 1: last_states.extend([last_states[0]] * (params.hist_size - 1)) else: assert len(last_states) == params.hist_size + 1 del last_states[0] # return the screen and the game features return screen, game_features
class DoomEnv(gym.Env, EzPickle): metadata = { 'render.modes': ['human', 'rgb_array'], 'video.frames_per_second': 35 } def __init__(self, level='deathmatch', obs_type='ram'): # super(DoomEnv, self).__init__() EzPickle.__init__(self, level.split('.')[0], obs_type) assert obs_type in ('ram', 'image') level = level.split('.')[0] Config.init(level) self.curr_seed = 0 self.game = DoomGame() self.lock = (DoomLock()).get_lock() self.level = level self.obs_type = obs_type self.tick = 4 self._mode = 'algo' self.is_render_in_human_mode = True self.is_game_initialized = False self.is_level_loaded = False self.viewer = None self.set_game(self.level, resolution=None, render=True) print() # todo: add frame skip option by using tick def step(self, action): reward = 0.0 # self.tick = 4 if self._mode == 'algo': if self.tick: reward = self.game.make_action(action, self.tick) else: reward = self.game.make_action(action) # self.game.set_action(action) # self.game.advance_action(4) # reward = self.game.get_last_reward() return self.get_obs(), reward, self.isDone(), self.get_info() def reset(self): if not self.is_game_initialized: self.__load_level() self.__init_game() self.__start_episode() return self.get_obs() def render(self, mode='human', **kwargs): if 'close' in kwargs and kwargs['close']: if self.viewer is not None: self.viewer.close() self.viewer = None return if mode == 'human' and not self.is_render_in_human_mode: return img = self.get_image() if mode == 'rgb_array': return img elif mode is 'human': if self.viewer is None: self.viewer = rendering.SimpleImageViewer() self.viewer.imshow(img) def close(self): with self.lock: self.game.close() def seed(self, seed=None): self.curr_seed = seeding.hash_seed(seed) % 2**32 return [self.curr_seed] # ================================== GETTERS SETTERS =============================================================== def set_game(self, level, resolution, render): self.__configure() self.__load_level(level) self.__set_resolution(resolution) self.__set_obs_and_ac_space() self.__set_player(render) def __configure(self, lock=None, **kwargs): self.seed() if lock is not None: self.lock = lock def __load_level(self, level=None): if level is not None: self.level = level.split('.')[0] self.is_level_loaded = False if self.is_level_loaded: return if self.is_game_initialized: self.is_game_initialized = False self.game.close() self.game = DoomGame() if not self.is_game_initialized: self.game.set_vizdoom_path(Config.VIZDOOM_PATH) self.game.set_doom_game_path(Config.FREEDOOM_PATH) # Common settings self.record_file_path = Config.RECORD_FILE_PATH self.game.load_config(Config.VIZDOOM_SCENARIO_PATH + Config.DOOM_SETTINGS[self.level][Config.CONFIG]) self.game.set_doom_scenario_path( Config.VIZDOOM_SCENARIO_PATH + Config.DOOM_SETTINGS[self.level][Config.SCENARIO]) if Config.DOOM_SETTINGS[self.level][Config.MAP] != '': self.game.set_doom_map( Config.DOOM_SETTINGS[self.level][Config.MAP]) self.game.set_doom_skill( Config.DOOM_SETTINGS[self.level][Config.DIFFICULTY]) self.allowed_actions = Config.DOOM_SETTINGS[self.level][Config.ACTIONS] self.available_game_variables = Config.DOOM_SETTINGS[self.level][ Config.GAME_VARIABLES] self.is_level_loaded = True def __set_resolution(self, resolution=None): if resolution is None: resolution = Config.DEFAULT_SCREEN_RESOLUTION resolution_l = resolution.lower() if resolution_l not in resolutions: raise gym.error.Error( 'Error - The specified resolution "{}" is not supported by Vizdoom.\n The list of valid' 'resolutions: {}'.format(resolution, resolutions)) if '_' in resolution_l: resolution_l = resolution_l.split('_')[1] self.scr_width = int(resolution_l.split("x")[0]) self.scr_height = int(resolution_l.split("x")[1]) self.game.set_screen_resolution( getattr(ScreenResolution, 'RES_{}X{}'.format(self.scr_width, self.scr_height))) self.screen_format = self.game.get_screen_format() self.screen_height = self.game.get_screen_height() self.screen_width = self.game.get_screen_width() def __set_obs_and_ac_space(self): if self.obs_type == 'ram': self.observation_space = spaces.Box( low=0, high=255, dtype=np.uint8, shape=(len(self.available_game_variables), )) elif self.obs_type == 'image': # self.observation_space = self.screen_resized self.observation_space = spaces.Box(low=0, high=255, shape=(self.scr_height, self.scr_width, 3), dtype=np.uint8) else: raise error.Error('Unrecognized observation type: {}'.format( self.obs_type)) if self.screen_format in inverted_screen_formats: self.dummy_screen = np.zeros(shape=(3, self.scr_height, self.scr_width), dtype=np.uint8) else: self.dummy_screen = np.zeros(shape=(self.scr_height, self.scr_width, 3), dtype=np.uint8) self.dummy_ram = [0] * len(self.available_game_variables) self.available_action_codes = [ list(a) for a in it.product([0, 1], repeat=self.game.get_available_buttons_size()) ] # self.__delete_conflict_actions() self.action_space = spaces.MultiDiscrete( [len(self.available_action_codes)]) def __set_player(self, render=True): self.game.set_window_visible(render) self.game.set_mode(Mode.PLAYER) def __init_game(self): try: with self.lock: self.game.init() self.is_game_initialized = True except (ViZDoomUnexpectedExitException, ViZDoomErrorException): raise error.Error('Could not start the game.') def __start_episode(self): if self.curr_seed > 0: self.game.set_seed(self.curr_seed) self.curr_seed = 0 if self.record_file_path: self.game.new_episode(self.record_file_path) else: self.game.new_episode() return def getState(self): return self.game.get_state() def getLastAction(self): return self.game.get_last_action() def getButtonsNames(self, action): return action_to_buttons(self.allowed_actions, action) def get_info(self): info = { "LEVEL": self.level, "TOTAL_REWARD": round(self.game.get_total_reward(), 4) } state_variables = self.get_ram() for i in range(len(self.available_game_variables)): info[self.available_game_variables[i]] = state_variables[i] return info def get_ram(self): if not self.is_game_initialized: raise NotImplementedError( "The game was not initialized. Run env.reset() first!") try: ram = self.getState().game_variables except AttributeError: ram = self.dummy_ram return ram def get_image(self): try: screen = self.getState().screen_buffer.copy() except AttributeError: screen = self.dummy_screen return self.invert_screen(screen) def get_obs(self): if self.obs_type == 'ram': return self.get_ram() elif self.obs_type == 'image': return self.get_image() def isDone(self): return self.game.is_episode_finished() or self.game.is_player_dead( ) or self.getState() is None # =========================================== ============================================================== def invert_screen(self, img): if self.screen_format in inverted_screen_formats: return np.rollaxis(img, 0, 3) else: return img def __delete_conflict_actions(self): if self._mode == 'human': return action_codes_copy = self.available_action_codes.copy() print("Initial actions size: " + str(len(action_codes_copy))) for i in tqdm.trange(len(self.available_action_codes)): action = self.available_action_codes[i] ac_names = action_to_buttons(self.allowed_actions, action) if all(elem in ac_names for elem in ['MOVE_LEFT', 'MOVE_RIGHT']) or all( elem in ac_names for elem in ['MOVE_BACKWARD', 'MOVE_FORWARD']) or all( elem in ac_names for elem in ['TURN_RIGHT', 'TURN_LEFT']) or all( elem in ac_names for elem in ['SELECT_NEXT_WEAPON', 'SELECT_PREV_WEAPON']): action_codes_copy.remove(action) print("Final actions size: " + str(len(action_codes_copy))) self.available_action_codes = action_codes_copy def __initHumanPlayer(self): self._mode = 'human' self.__load_level() self.game.add_game_args('+freelook 1') self.game.set_window_visible(True) self.game.set_mode(Mode.SPECTATOR) self.is_render_in_human_mode = False self.__init_game() def advanceAction(self, tick=0): try: if tick: self.game.advance_action(tick) else: self.game.advance_action() return True except ViZDoomUnexpectedExitException: return False def playHuman(self): self.__initHumanPlayer() while not self.game.is_episode_finished( ) and not self.game.is_player_dead(): self.advanceAction() state = self.getState() if state is None: if self.record_file_path is None: self.game.new_episode() else: self.game.new_episode(self.record_file_path) state = self.getState() total_reward = self.game.get_total_reward() info = self.get_info() info["TOTAL_REWARD"] = round(total_reward, 4) print('===============================') print('State: #' + str(state.number)) print('Action: \t' + str(self.game.get_last_action()) + '\t (=> only allowed actions)') print('Reward: \t' + str(self.game.get_last_reward())) print('Total Reward: \t' + str(total_reward)) print('Variables: \n' + str(info)) sleep(0.02857) # 35 fps = 0.02857 sleep between frames print('===============================') print('Done') return
life_buffer = [] while not game.is_episode_finished(): loss = 0 r_t = 0 a_t = np.zeros([action_size]) # Epsilon Greedy action_idx = agent.get_action(s_t, m_t, goal, inference_goal) a_t[action_idx] = 1 a_t = a_t.astype(int) game.set_action(a_t.tolist()) skiprate = agent.frame_per_action game.advance_action(skiprate) game_state = game.get_state() # Observe again after we take the action is_terminated = game.is_episode_finished() r_t = game.get_last_reward() if (is_terminated): if (life > max_life): max_life = life GAME += 1 life_buffer.append(life) print ("Episode Finish ") game.new_episode() game_state = game.get_state() misc = game_state.game_variables
class DoomScenario: """ DoomScenario class runs instances of Vizdoom according to scenario configuration (.cfg) files. Scenario Configuration files for this project are located in the /src/configs/ folder. """ def __init__(self, config_filename): ''' Method initiates Vizdoom with desired configuration file. ''' self.config_filename = config_filename self.game = DoomGame() self.game.load_config("configs/" + config_filename) self.game.set_window_visible(False) self.game.init() self.res = (self.game.get_screen_height(), self.game.get_screen_width()) self.actions = [ list(a) for a in it.product([0, 1], repeat=self.game.get_available_buttons_size()) ] self.pbar = None self.game.new_episode() def play(self, action, tics): ''' Method advances state with desired action for a number of tics. ''' self.game.set_action(action) self.game.advance_action(tics, True) if self.pbar: self.pbar.update(int(tics)) def get_processed_state(self, depth_radius, depth_contrast): ''' Method processes the Vizdoom RGB and depth buffer into a composite one channel image that can be used by the Models. depth_radius defines how far the depth buffer sees with 1.0 being as far as ViZDoom allows. depth_contrast defines how much of the depth buffer is in the final processed image as compared to the greyscaled RGB buffer. **processed = (1-depth_contrast)* grey_buffer + depth_contrast*depth_buffer ''' state = self.game.get_state() if not self.game.is_episode_finished(): img = state.screen_buffer # screen pixels # print(img) screen_buffer = np.array(img).astype('float32') / 255 # print(screen_buffer.shape) # (3, 120, 160) try: # Grey Scaling grey_buffer = np.dot(np.transpose(screen_buffer, (1, 2, 0)), [0.21, 0.72, 0.07]) # print(grey_buffer.shape) # (120, 160) # Depth Radius depth_buffer = np.array(state.depth_buffer).astype('float32') / 255 depth_buffer[(depth_buffer > depth_radius)] = depth_radius #Effects depth radius depth_buffer_filtered = (depth_buffer - np.amin(depth_buffer)) / ( np.amax(depth_buffer) - np.amin(depth_buffer)) # Depth Contrast processed_buffer = ( (1 - depth_contrast) * grey_buffer) + (depth_contrast * (1 - depth_buffer)) processed_buffer = (processed_buffer - np.amin(processed_buffer) ) / (np.amax(processed_buffer) - np.amin(processed_buffer)) processed_buffer = np.round(processed_buffer, 6) processed_buffer = processed_buffer.reshape(self.res[-2:]) except: processed_buffer = np.zeros(self.res[-2:]) return processed_buffer # balance the depth & RGB data def run(self, agent, save_replay='', verbose=False, return_data=False): ''' Method runs a instance of DoomScenario. ''' if return_data: data_S = [] data_a = [] if verbose: print("\nRunning Simulation:", self.config_filename) self.pbar = tqdm(total=self.game.get_episode_timeout()) # Initiate New Instance self.game.close() self.game.set_window_visible(False) self.game.add_game_args("+vid_forcesurface 1 ") self.game.init() if save_replay != '': self.game.new_episode("../data/replay_data/" + save_replay) else: self.game.new_episode() # Run Simulation while not self.game.is_episode_finished(): S = agent.get_state_data(self) q = agent.model.online_network.predict(S) if np.random.random() < 0.1: q = np.random.choice(len(q[0]), 1, p=softmax(q[0], 1))[0] else: q = int(np.argmax(q[0])) a = agent.model.predict(self, q) if return_data: delta = np.zeros((len(self.actions))) a_ = np.cast['int'](a) delta[a_] = 1 data_S.append(S.reshape(S.shape[1], S.shape[2], S.shape[3])) data_a.append(delta) if not self.game.is_episode_finished(): self.play(a, agent.frame_skips + 1) if agent.model.__class__.__name__ == 'HDQNModel' and not self.game.is_episode_finished( ): if q >= len(agent.model.actions): for i in range(agent.model.skill_frame_skip): if not self.game.is_episode_finished(): a = agent.model.predict(self, q) self.play(a, agent.frame_skips + 1) else: break # Reset Agent and Return Score agent.frames = None if agent.model.__class__.__name__ == 'HDQNModel': agent.model.sub_model_frames = None score = self.game.get_total_reward() if verbose: self.pbar.close() print("Total Score:", score) if return_data: data_S = np.array(data_S) data_a = np.array(data_a) return [data_S, data_a] return score def replay(self, filename, verbose=False, doom_like=False): ''' Method runs a replay of the simulations at 800 x 600 resolution. ''' print("\nRunning Replay:", filename) # Initiate Replay self.game.close() self.game.set_screen_resolution(ScreenResolution.RES_800X600) self.game.set_window_visible(True) self.game.add_game_args("+vid_forcesurface 1") if doom_like: self.game.set_render_hud(True) self.game.set_render_minimal_hud(False) self.game.set_render_crosshair(False) self.game.set_render_weapon(True) self.game.set_render_particles(True) self.game.init() self.game.replay_episode("../data/replay_data/" + filename) # Run Replay while not self.game.is_episode_finished(): if verbose: print("Reward:", self.game.get_last_reward()) self.game.advance_action() # Print Score score = self.game.get_total_reward() print("Total Score:", score) self.game.close() def apprentice_run(self, test=False): ''' Method runs an apprentice data gathering. ''' # Initiate New Instance self.game.close() self.game.set_mode(Mode.SPECTATOR) self.game.set_screen_resolution(ScreenResolution.RES_800X600) self.game.set_window_visible(True) self.game.set_ticrate(30) self.game.init() self.game.new_episode() # Run Simulation while not self.game.is_episode_finished(): self.game.advance_action() self.game.close()
class ViZDoom(Environment): """ [ViZDoom](https://github.com/mwydmuch/ViZDoom) environment adapter (specification key: `vizdoom`). Args: level (string): ViZDoom configuration file (<span style="color:#C00000"><b>required</b></span>). include_variables (bool): Whether to include game variables to state (<span style="color:#00C000"><b>default</b></span>: false). factored_action (bool): Whether to use factored action representation (<span style="color:#00C000"><b>default</b></span>: false). visualize (bool): Whether to visualize interaction (<span style="color:#00C000"><b>default</b></span>: false). frame_skip (int > 0): Number of times to repeat an action without observing (<span style="color:#00C000"><b>default</b></span>: 12). seed (int): Random seed (<span style="color:#00C000"><b>default</b></span>: none). """ def __init__(self, level, visualize=False, include_variables=False, factored_action=False, frame_skip=12, seed=None): from vizdoom import DoomGame, Mode, ScreenFormat, ScreenResolution self.config_file = level self.include_variables = include_variables self.factored_action = factored_action self.visualize = visualize self.frame_skip = frame_skip self.environment = DoomGame() self.environment.load_config(self.config_file) if self.visualize: self.environment.set_window_visible(True) self.environment.set_mode(Mode.ASYNC_PLAYER) else: self.environment.set_window_visible(False) self.environment.set_mode(Mode.PLAYER) # e.g. CRCGCB, RGB24, GRAY8 self.environment.set_screen_format(ScreenFormat.RGB24) # e.g. RES_320X240, RES_640X480, RES_1920X1080 self.environment.set_screen_resolution(ScreenResolution.RES_640X480) self.environment.set_depth_buffer_enabled(False) self.environment.set_labels_buffer_enabled(False) self.environment.set_automap_buffer_enabled(False) if seed is not None: self.environment.setSeed(seed) self.environment.init() self.state_shape = (640, 480, 3) self.num_variables = self.environment.get_available_game_variables_size( ) self.num_buttons = self.environment.get_available_buttons_size() self.actions = [ tuple(a) for a in itertools.product([0, 1], repeat=self.num_buttons) ] def __str__(self): return super().__str__() + '({})'.format(self.config_file) def states(self): if self.include_variables: return OrderedDict(screen=dict(type='float', shape=self.state_shape), variables=dict(type='float', shape=self.num_variables)) else: return dict(type='float', shape=self.state_shape) def actions(self): if self.factored_action: return dict(type='bool', shape=self.num_buttons) else: return dict(type='int', shape=(), num_values=len(self.actions)) def close(self): self.environment.close() self.environment = None def get_states(self): state = self.environment.get_state() screen = state.screen_buffer.astype(dtype=np.float32) / 255.0 if self.include_variables: return OrderedDict(screen=screen, variables=state.game_variables) else: return screen def reset(self): self.environment.new_episode() return self.get_states() def execute(self, actions): if self.factored_action: action = np.where(actions, 1.0, 0.0) else: action = self.actions[actions] if self.visualize: self.environment.set_action(action) reward = 0.0 for _ in range(self.frame_skip): self.environment.advance_action() reward += self.environment.get_last_reward() else: reward = self.environment.make_action(action, self.frame_skip) terminal = self.environment.is_episode_finished() states = self.get_states() return states, terminal, reward